# Copyright 2012 Hewlett-Packard Development Company, L.P. # Copyright 2021-2022 Acme Gating, LLC # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. import configparser import gc import json import logging import os import re import shutil import socket import textwrap import threading import time from collections import namedtuple from unittest import mock, skip from uuid import uuid4 from kazoo.exceptions import NoNodeError from testtools.matchers import StartsWith import git import fixtures import zuul.change_matcher from zuul.driver.gerrit import gerritreporter import zuul.scheduler import zuul.model import zuul.merger.merger from zuul.lib import yamlutil as yaml from tests.base import ( SSLZuulTestCase, ZuulTestCase, repack_repo, simple_layout, iterate_timeout, RecordingExecutorServer, TestConnectionRegistry, FIXTURE_DIR, skipIfMultiScheduler, ) from zuul.zk.change_cache import ChangeKey from zuul.zk.event_queues import PIPELINE_NAME_ROOT from zuul.zk.layout import LayoutState from zuul.zk.locks import management_queue_lock, pipeline_lock from zuul.zk import zkobject EMPTY_LAYOUT_STATE = LayoutState("", "", 0, None, {}, -1) class TestSchedulerSSL(SSLZuulTestCase): tenant_config_file = 'config/single-tenant/main.yaml' def test_jobs_executed(self): "Test that jobs are executed and a change is merged" A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') A.addApproval('Code-Review', 2) self.fake_gerrit.addEvent(A.addApproval('Approved', 1)) self.waitUntilSettled() self.assertEqual(self.getJobFromHistory('project-merge').result, 'SUCCESS') self.assertEqual(self.getJobFromHistory('project-test1').result, 'SUCCESS') self.assertEqual(self.getJobFromHistory('project-test2').result, 'SUCCESS') self.assertEqual(A.data['status'], 'MERGED') self.assertEqual(A.reported, 2) self.assertEqual(self.getJobFromHistory('project-test1').node, 'label1') self.assertEqual(self.getJobFromHistory('project-test2').node, 'label1') class TestSchedulerZone(ZuulTestCase): tenant_config_file = 'config/single-tenant/main.yaml' def setUp(self): super(TestSchedulerZone, self).setUp() self.fake_nodepool.attributes = {'executor-zone': 'test-provider.vpn'} # Create an unzoned executor config = configparser.ConfigParser() config.read_dict(self.config) config.remove_option('executor', 'zone') config.set('executor', 'command_socket', os.path.join(self.test_root, 'executor2.socket')) executor_connections = TestConnectionRegistry( self.changes, self.config, self.additional_event_queues, self.upstream_root, self.poller_events, self.git_url_with_auth, self.addCleanup) executor_connections.configure(self.config, source_only=True) self.executor_server_unzoned = RecordingExecutorServer( config, connections=executor_connections, jobdir_root=self.jobdir_root, _run_ansible=self.run_ansible, _test_root=self.test_root, log_console_port=self.log_console_port) self.executor_server_unzoned.start() self.addCleanup(self._shutdown_executor) def _shutdown_executor(self): self.executor_server_unzoned.hold_jobs_in_build = False self.executor_server_unzoned.release() self.executor_server_unzoned.stop() self.executor_server_unzoned.join() def setup_config(self, config_file: str): config = super(TestSchedulerZone, self).setup_config(config_file) config.set('executor', 'zone', 'test-provider.vpn') return config def test_jobs_executed(self): "Test that jobs are executed and a change is merged per zone" # Validate that the reported executor stats are correct. There must # be two executors online (one unzoned and one zoned) # TODO(corvus): remove deprecated top-level stats in 5.0 self.assertReportedStat( 'zuul.executors.online', value='2', kind='g') self.assertReportedStat( 'zuul.executors.unzoned.online', value='1', kind='g') self.assertReportedStat( 'zuul.executors.zone.test-provider_vpn.online', value='1', kind='g') self.hold_jobs_in_queue = True A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') A.addApproval('Code-Review', 2) self.fake_gerrit.addEvent(A.addApproval('Approved', 1)) self.waitUntilSettled() queue = list(self.executor_api.queued()) self.assertEqual(len(self.builds), 0) self.assertEqual(len(queue), 1) self.assertEqual('test-provider.vpn', queue[0].zone) self.hold_jobs_in_queue = False self.executor_api.release() self.waitUntilSettled() self.assertEqual(self.getJobFromHistory('project-merge').result, 'SUCCESS') self.assertEqual(self.getJobFromHistory('project-test1').result, 'SUCCESS') self.assertEqual(self.getJobFromHistory('project-test2').result, 'SUCCESS') self.assertEqual(A.data['status'], 'MERGED') self.assertEqual(A.reported, 2) self.assertEqual(self.getJobFromHistory('project-test1').node, 'label1') self.assertEqual(self.getJobFromHistory('project-test2').node, 'label1') # Validate that both (zoned and unzoned) executors are accepting work self.assertReportedStat( 'zuul.executors.accepting', value='2', kind='g') self.assertReportedStat( 'zuul.executors.unzoned.accepting', value='1', kind='g') self.assertReportedStat( 'zuul.executors.zone.test-provider_vpn.accepting', value='1', kind='g') def test_executor_disconnect(self): "Test that jobs are completed after an executor disconnect" self.executor_server.hold_jobs_in_build = True A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') A.addApproval('Code-Review', 2) self.fake_gerrit.addEvent(A.addApproval('Approved', 1)) self.waitUntilSettled() # Forcibly disconnect the executor from ZK self.executor_server.zk_client.client.stop() self.executor_server.zk_client.client.start() # Find the build in the scheduler so we can check its status tenant = self.scheds.first.sched.abide.tenants.get('tenant-one') items = tenant.layout.pipelines['gate'].getAllItems() builds = items[0].current_build_set.getBuilds() build = builds[0] # Clean up the build self.scheds.first.sched.executor.cleanupLostBuildRequests() # Wait for the build to be reported as lost for x in iterate_timeout(30, 'retry build'): if build.result == 'RETRY': break # If we didn't timeout, then it worked; we're done self.executor_server.hold_jobs_in_build = False self.executor_server.release() self.waitUntilSettled() # There is a test-only race in the recording executor class # where we may record a successful first build, even though # the executor didn't actually send a build complete event. # This could probabyl be improved, but for now, it's # sufficient to verify that the job was retried. So we omit a # result classifier on the first build. self.assertHistory([ dict(name='project-merge', changes='1,1'), dict(name='project-merge', result='SUCCESS', changes='1,1'), dict(name='project-test1', result='SUCCESS', changes='1,1'), dict(name='project-test2', result='SUCCESS', changes='1,1'), ], ordered=False) class TestSchedulerZoneFallback(ZuulTestCase): tenant_config_file = 'config/single-tenant/main.yaml' def setup_config(self, config_file: str): config = super().setup_config(config_file) config.set('executor', 'zone', 'test-provider.vpn') config.set('executor', 'allow_unzoned', 'true') return config def test_jobs_executed(self): "Test that jobs are executed and a change is merged per zone" self.hold_jobs_in_queue = True # Validate that the reported executor stats are correct. Since # the executor accepts zoned and unzoned job it should be counted # in both metrics. self.assertReportedStat( 'zuul.executors.online', value='1', kind='g') self.assertReportedStat( 'zuul.executors.unzoned.online', value='1', kind='g') self.assertReportedStat( 'zuul.executors.zone.test-provider_vpn.online', value='1', kind='g') A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') A.addApproval('Code-Review', 2) self.fake_gerrit.addEvent(A.addApproval('Approved', 1)) self.waitUntilSettled() queue = list(self.executor_api.queued()) self.assertEqual(len(self.builds), 0) self.assertEqual(len(queue), 1) self.assertEqual(None, queue[0].zone) self.hold_jobs_in_queue = False self.executor_api.release() self.waitUntilSettled() self.assertEqual(self.getJobFromHistory('project-merge').result, 'SUCCESS') self.assertEqual(self.getJobFromHistory('project-test1').result, 'SUCCESS') self.assertEqual(self.getJobFromHistory('project-test2').result, 'SUCCESS') self.assertEqual(A.data['status'], 'MERGED') self.assertEqual(A.reported, 2) self.assertEqual(self.getJobFromHistory('project-test1').node, 'label1') self.assertEqual(self.getJobFromHistory('project-test2').node, 'label1') class TestSchedulerAutoholdHoldExpiration(ZuulTestCase): ''' This class of tests validates the autohold node expiration values are set correctly via zuul config or from a custom value. ''' config_file = 'zuul-hold-expiration.conf' tenant_config_file = 'config/single-tenant/main.yaml' @simple_layout('layouts/autohold.yaml') def test_autohold_max_hold_default(self): ''' Test that the hold request node expiration will default to the value specified in the configuration file. ''' # Add a autohold with no hold expiration. self.scheds.first.sched.autohold( 'tenant-one', 'review.example.com/org/project', 'project-test2', ".*", "reason text", 1, None) # There should be a record in ZooKeeper request_list = self.sched_zk_nodepool.getHoldRequests() self.assertEqual(1, len(request_list)) request = self.sched_zk_nodepool.getHoldRequest( request_list[0]) self.assertIsNotNone(request) self.assertEqual('tenant-one', request.tenant) self.assertEqual('review.example.com/org/project', request.project) self.assertEqual('project-test2', request.job) self.assertEqual('reason text', request.reason) self.assertEqual(1, request.max_count) self.assertEqual(0, request.current_count) self.assertEqual([], request.nodes) # This should be the default value from the zuul config file. self.assertEqual(1800, request.node_expiration) @simple_layout('layouts/autohold.yaml') def test_autohold_max_hold_custom(self): ''' Test that the hold request node expiration will be set to the custom value specified in the request. ''' # Add a autohold with a custom hold expiration. self.scheds.first.sched.autohold( 'tenant-one', 'review.example.com/org/project', 'project-test2', ".*", "reason text", 1, 500) # There should be a record in ZooKeeper request_list = self.sched_zk_nodepool.getHoldRequests() self.assertEqual(1, len(request_list)) request = self.sched_zk_nodepool.getHoldRequest( request_list[0]) self.assertIsNotNone(request) self.assertEqual('tenant-one', request.tenant) self.assertEqual('review.example.com/org/project', request.project) self.assertEqual('project-test2', request.job) self.assertEqual('reason text', request.reason) self.assertEqual(1, request.max_count) self.assertEqual(0, request.current_count) self.assertEqual([], request.nodes) # This should be the value from the user request. self.assertEqual(500, request.node_expiration) @simple_layout('layouts/autohold.yaml') def test_autohold_max_hold_custom_invalid(self): ''' Test that if the custom hold request node expiration is higher than our configured max, it will be lowered to the max. ''' # Add a autohold with a custom hold expiration that is higher than our # configured max. self.scheds.first.sched.autohold( 'tenant-one', 'review.example.com/org/project', 'project-test2', ".*", "reason text", 1, 10000) # There should be a record in ZooKeeper request_list = self.sched_zk_nodepool.getHoldRequests() self.assertEqual(1, len(request_list)) request = self.sched_zk_nodepool.getHoldRequest( request_list[0]) self.assertIsNotNone(request) self.assertEqual('tenant-one', request.tenant) self.assertEqual('review.example.com/org/project', request.project) self.assertEqual('project-test2', request.job) self.assertEqual('reason text', request.reason) self.assertEqual(1, request.max_count) self.assertEqual(0, request.current_count) self.assertEqual([], request.nodes) # This should be the max value from the zuul config file. self.assertEqual(3600, request.node_expiration) class TestScheduler(ZuulTestCase): tenant_config_file = 'config/single-tenant/main.yaml' def test_jobs_executed(self): "Test that jobs are executed and a change is merged" A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') A.addApproval('Code-Review', 2) self.fake_gerrit.addEvent(A.addApproval('Approved', 1)) self.waitUntilSettled() self.assertEqual(self.getJobFromHistory('project-merge').result, 'SUCCESS') self.assertEqual(self.getJobFromHistory('project-test1').result, 'SUCCESS') self.assertEqual(self.getJobFromHistory('project-test2').result, 'SUCCESS') self.assertEqual(A.data['status'], 'MERGED') self.assertEqual(A.reported, 2) self.assertEqual(self.getJobFromHistory('project-test1').node, 'label1') self.assertEqual(self.getJobFromHistory('project-test2').node, 'label1') self.assertThat(A.messages[1], StartsWith( 'Build succeeded (gate).\n' 'https://zuul.example.com/t/tenant-one/buildset')) # TODOv3(jeblair): we may want to report stats by tenant (also?). # Per-driver self.assertReportedStat('zuul.event.gerrit.comment-added', value='1', kind='c') # Per-driver per-connection self.assertReportedStat('zuul.event.gerrit.gerrit.comment-added', value='1', kind='c') self.assertReportedStat( 'zuul.tenant.tenant-one.trigger_events', value='0', kind='g') self.assertReportedStat( 'zuul.tenant.tenant-one.management_events', value='0', kind='g') self.assertReportedStat( 'zuul.tenant.tenant-one.pipeline.gate.current_changes', value='1', kind='g') self.assertReportedStat( 'zuul.tenant.tenant-one.pipeline.gate.window', value='20', kind='g') self.assertReportedStat( 'zuul.tenant.tenant-one.pipeline.gate.queue.' 'org.project.current_changes', value='1', kind='g') self.assertReportedStat( 'zuul.tenant.tenant-one.pipeline.gate.queue.org.project.window', value='20', kind='g') self.assertReportedStat( 'zuul.tenant.tenant-one.pipeline.gate.queue.org.project.window', value='21', kind='g') self.assertReportedStat( 'zuul.tenant.tenant-one.pipeline.gate.project.review_example_com.' 'org_project.master.job.project-merge.SUCCESS', kind='ms') self.assertReportedStat( 'zuul.tenant.tenant-one.pipeline.gate.project.review_example_com.' 'org_project.master.job.project-merge.SUCCESS', value='1', kind='c') self.assertReportedStat( 'zuul.tenant.tenant-one.pipeline.gate.resident_time', kind='ms') self.assertReportedStat( 'zuul.tenant.tenant-one.pipeline.gate.total_changes', value='1', kind='c') self.assertReportedStat( 'zuul.tenant.tenant-one.pipeline.gate.queue.' 'org.project.resident_time', kind='ms') self.assertReportedStat( 'zuul.tenant.tenant-one.pipeline.gate.queue.' 'org.project.total_changes', value='1', kind='c') self.assertReportedStat( 'zuul.tenant.tenant-one.pipeline.gate.trigger_events', value='0', kind='g') self.assertReportedStat( 'zuul.tenant.tenant-one.pipeline.gate.result_events', value='0', kind='g') self.assertReportedStat( 'zuul.tenant.tenant-one.pipeline.gate.management_events', value='0', kind='g') self.assertReportedStat( 'zuul.tenant.tenant-one.pipeline.gate.project.review_example_com.' 'org_project.master.resident_time', kind='ms') self.assertReportedStat( 'zuul.tenant.tenant-one.pipeline.gate.project.review_example_com.' 'org_project.master.total_changes', value='1', kind='c') exec_key = 'zuul.executor.%s' % self.executor_server.hostname.replace( '.', '_') self.assertReportedStat(exec_key + '.builds', value='1', kind='c') self.assertReportedStat(exec_key + '.starting_builds', kind='g') self.assertReportedStat(exec_key + '.starting_builds', kind='ms') self.assertReportedStat( 'zuul.nodepool.requests.requested.total', value='1', kind='c') self.assertReportedStat( 'zuul.nodepool.requests.requested.label.label1', value='1', kind='c') self.assertReportedStat( 'zuul.nodepool.requests.fulfilled.label.label1', value='1', kind='c') self.assertReportedStat( 'zuul.nodepool.requests.requested.size.1', value='1', kind='c') self.assertReportedStat( 'zuul.nodepool.requests.fulfilled.size.1', value='1', kind='c') # just check for existence, since we can not know if a request is # in-flight during the sched._stats_inverval self.assertReportedStat( 'zuul.nodepool.current_requests', kind='g') self.assertReportedStat( 'zuul.nodepool.tenant.tenant-one.current_requests', kind='g') self.assertReportedStat( 'zuul.executors.online', value='1', kind='g') self.assertReportedStat( 'zuul.executors.accepting', value='1', kind='g') self.assertReportedStat( 'zuul.mergers.online', value='1', kind='g') self.assertReportedStat('zuul.scheduler.eventqueues.connection.gerrit', value='0', kind='g') self.assertReportedStat('zuul.scheduler.run_handler', kind='ms') # Catch time / monotonic errors for key in [ 'zuul.tenant.tenant-one.event_enqueue_processing_time', 'zuul.tenant.tenant-one.event_enqueue_time', 'zuul.tenant.tenant-one.reconfiguration_time', 'zuul.tenant.tenant-one.pipeline.gate.event_enqueue_time', 'zuul.tenant.tenant-one.pipeline.gate.merge_request_time', 'zuul.tenant.tenant-one.pipeline.gate.merger_merge_op_time', 'zuul.tenant.tenant-one.pipeline.gate.job_freeze_time', 'zuul.tenant.tenant-one.pipeline.gate.node_request_time', 'zuul.tenant.tenant-one.pipeline.gate.job_wait_time', 'zuul.tenant.tenant-one.pipeline.gate.event_job_time', 'zuul.tenant.tenant-one.pipeline.gate.resident_time', 'zuul.tenant.tenant-one.pipeline.gate.read_time', 'zuul.tenant.tenant-one.pipeline.gate.write_time', 'zuul.tenant.tenant-one.pipeline.gate.process', 'zuul.tenant.tenant-one.pipeline.gate.event_process', 'zuul.tenant.tenant-one.pipeline.gate.handling', 'zuul.tenant.tenant-one.pipeline.gate.refresh', ]: val = self.assertReportedStat(key, kind='ms') self.assertTrue(0.0 < float(val) < 60000.0) for key in [ 'zuul.tenant.tenant-one.pipeline.gate.read_objects', 'zuul.tenant.tenant-one.pipeline.gate.write_objects', 'zuul.tenant.tenant-one.pipeline.gate.read_znodes', 'zuul.tenant.tenant-one.pipeline.gate.write_znodes', 'zuul.tenant.tenant-one.pipeline.gate.write_bytes', ]: # 'zuul.tenant.tenant-one.pipeline.gate.read_bytes' is # expected to be zero since it's initialized after reading val = self.assertReportedStat(key, kind='g') self.assertTrue(0.0 < float(val) < 60000.0) self.assertReportedStat('zuul.tenant.tenant-one.pipeline.gate.' 'data_size_compressed', kind='g') self.assertReportedStat('zuul.tenant.tenant-one.pipeline.gate.' 'data_size_uncompressed', kind='g') self.assertReportedStat('zuul.connection.gerrit.cache.' 'data_size_compressed', kind='g') self.assertReportedStat('zuul.connection.gerrit.cache.' 'data_size_uncompressed', kind='g') for build in self.history: self.assertTrue(build.parameters['zuul']['voting']) def test_zk_profile(self): command_socket = self.scheds.first.sched.config.get( 'scheduler', 'command_socket') A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') zplog = logging.getLogger('zuul.profile') with self.assertLogs('zuul.profile', level='DEBUG') as logs: self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) self.waitUntilSettled() # assertNoLogs doesn't appear until py3.10, so we need to # emit a single log line in order to assert that there # aren't any others. zplog.debug('test') self.assertEqual(1, len(logs.output)) args = json.dumps(['tenant-one', 'check']) with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as s: s.connect(command_socket) s.sendall(f'zkprofile {args}\n'.encode('utf8')) with self.assertLogs('zuul.profile', level='DEBUG'): self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) self.waitUntilSettled() with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as s: s.connect(command_socket) s.sendall(f'zkprofile {args}\n'.encode('utf8')) with self.assertLogs('zuul.profile', level='DEBUG') as logs: self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) self.waitUntilSettled() zplog.debug('test') self.assertEqual(1, len(logs.output)) def test_initial_pipeline_gauges(self): "Test that each pipeline reported its length on start" self.assertReportedStat('zuul.tenant.tenant-one.pipeline.gate.' 'current_changes', value='0', kind='g') self.assertReportedStat('zuul.tenant.tenant-one.pipeline.check.' 'current_changes', value='0', kind='g') def test_job_branch(self): "Test the correct variant of a job runs on a branch" self.create_branch('org/project', 'stable') self.fake_gerrit.addEvent( self.fake_gerrit.getFakeBranchCreatedEvent( 'org/project', 'stable')) self.waitUntilSettled() A = self.fake_gerrit.addFakeChange('org/project', 'stable', 'A') A.addApproval('Code-Review', 2) self.fake_gerrit.addEvent(A.addApproval('Approved', 1)) self.waitUntilSettled() self.assertEqual(self.getJobFromHistory('project-test1').result, 'SUCCESS') self.assertEqual(self.getJobFromHistory('project-test2').result, 'SUCCESS') self.assertEqual(A.data['status'], 'MERGED') self.assertEqual(A.reported, 2, "A should report start and success") self.assertIn('gate', A.messages[1], "A should transit gate") self.assertEqual(self.getJobFromHistory('project-test1').node, 'label2') @simple_layout('layouts/branch-deletion.yaml') def test_branch_deletion(self): "Test the correct variant of a job runs on a branch" # Start a secondary merger so this test exercises branch # deletion on both a merger and a separate executor. self.executor_server._merger_running = False self.executor_server.merger_loop_wake_event.set() self.executor_server.merger_thread.join() self._startMerger() self.create_branch('org/project', 'stable') self.fake_gerrit.addEvent( self.fake_gerrit.getFakeBranchCreatedEvent( 'org/project', 'stable')) self.waitUntilSettled() A = self.fake_gerrit.addFakeChange('org/project', 'stable', 'A') self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) self.waitUntilSettled() self.assertEqual(self.getJobFromHistory('project-test2').result, 'SUCCESS') self.delete_branch('org/project', 'stable') path = os.path.join(self.executor_src_root, 'review.example.com') shutil.rmtree(path) self.executor_server.hold_jobs_in_build = True A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) self.waitUntilSettled() build = self.builds[0] # Make sure there is no stable branch in the checked out git repo. pname = 'review.example.com/org/project' work = build.getWorkspaceRepos([pname]) work = work[pname] heads = set([str(x) for x in work.heads]) self.assertEqual(heads, set(['master'])) self.executor_server.hold_jobs_in_build = False build.release() self.waitUntilSettled() self.assertEqual(self.getJobFromHistory('project-test1').result, 'SUCCESS') @simple_layout('layouts/branch-repo-state.yaml') def test_branch_repo_state(self): """ Test that we schedule no initial merge for branch/ref events and the the global repo state only if there are jobs to run. """ del self.merge_job_history A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A') A.setMerged() self.fake_gerrit.addEvent(A.getRefUpdatedEvent()) self.waitUntilSettled() # No merge expected as there are not jobs for project1 in the # post pipeline. self.assertIsNone(self.merge_job_history.get( zuul.model.MergeRequest.REF_STATE)) B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B') B.setMerged() self.fake_gerrit.addEvent(B.getRefUpdatedEvent()) self.waitUntilSettled() # One refstate merge request for the global repo state since # there are jobs configured for project2 in the post pipeline. refstate_jobs = self.merge_job_history.get( zuul.model.MergeRequest.REF_STATE) self.assertEqual(len(refstate_jobs), 1) merge_request = refstate_jobs[0] merge_items = {i["project"] for i in merge_request.payload["items"]} self.assertEqual( merge_items, {"org/project1", "org/project2", "org/common-config"}) def test_parallel_changes(self): "Test that changes are tested in parallel and merged in series" self.executor_server.hold_jobs_in_build = True A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B') C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C') A.addApproval('Code-Review', 2) B.addApproval('Code-Review', 2) C.addApproval('Code-Review', 2) self.fake_gerrit.addEvent(A.addApproval('Approved', 1)) self.fake_gerrit.addEvent(B.addApproval('Approved', 1)) self.fake_gerrit.addEvent(C.addApproval('Approved', 1)) self.waitUntilSettled() self.assertEqual(len(self.builds), 1) self.assertEqual(self.builds[0].name, 'project-merge') self.assertTrue(self.builds[0].hasChanges(A)) self.executor_server.release('.*-merge') self.waitUntilSettled() self.assertEqual(len(self.builds), 3) self.assertEqual(self.builds[0].name, 'project-test1') self.assertTrue(self.builds[0].hasChanges(A)) self.assertEqual(self.builds[1].name, 'project-test2') self.assertTrue(self.builds[1].hasChanges(A)) self.assertEqual(self.builds[2].name, 'project-merge') self.assertTrue(self.builds[2].hasChanges(A, B)) self.executor_server.release('.*-merge') self.waitUntilSettled() self.assertEqual(len(self.builds), 5) self.assertEqual(self.builds[0].name, 'project-test1') self.assertTrue(self.builds[0].hasChanges(A)) self.assertEqual(self.builds[1].name, 'project-test2') self.assertTrue(self.builds[1].hasChanges(A)) self.assertEqual(self.builds[2].name, 'project-test1') self.assertTrue(self.builds[2].hasChanges(A, B)) self.assertEqual(self.builds[3].name, 'project-test2') self.assertTrue(self.builds[3].hasChanges(A, B)) self.assertEqual(self.builds[4].name, 'project-merge') self.assertTrue(self.builds[4].hasChanges(A, B, C)) self.executor_server.release('.*-merge') self.waitUntilSettled() self.assertEqual(len(self.builds), 6) self.assertEqual(self.builds[0].name, 'project-test1') self.assertTrue(self.builds[0].hasChanges(A)) self.assertEqual(self.builds[1].name, 'project-test2') self.assertTrue(self.builds[1].hasChanges(A)) self.assertEqual(self.builds[2].name, 'project-test1') self.assertTrue(self.builds[2].hasChanges(A, B)) self.assertEqual(self.builds[3].name, 'project-test2') self.assertTrue(self.builds[3].hasChanges(A, B)) self.assertEqual(self.builds[4].name, 'project-test1') self.assertTrue(self.builds[4].hasChanges(A, B, C)) self.assertEqual(self.builds[5].name, 'project-test2') self.assertTrue(self.builds[5].hasChanges(A, B, C)) self.executor_server.hold_jobs_in_build = False self.executor_server.release() self.waitUntilSettled() self.assertEqual(len(self.builds), 0) self.assertEqual(len(self.history), 9) self.assertEqual(A.data['status'], 'MERGED') self.assertEqual(B.data['status'], 'MERGED') self.assertEqual(C.data['status'], 'MERGED') self.assertEqual(A.reported, 2) self.assertEqual(B.reported, 2) self.assertEqual(C.reported, 2) def test_failed_changes(self): "Test that a change behind a failed change is retested" self.executor_server.hold_jobs_in_build = True A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B') A.addApproval('Code-Review', 2) B.addApproval('Code-Review', 2) self.executor_server.failJob('project-test1', A) self.fake_gerrit.addEvent(A.addApproval('Approved', 1)) self.fake_gerrit.addEvent(B.addApproval('Approved', 1)) self.waitUntilSettled() self.assertBuilds([dict(name='project-merge', changes='1,1')]) self.executor_server.release('.*-merge') self.waitUntilSettled() # A/project-merge is complete self.assertBuilds([ dict(name='project-test1', changes='1,1'), dict(name='project-test2', changes='1,1'), dict(name='project-merge', changes='1,1 2,1'), ]) self.executor_server.release('.*-merge') self.waitUntilSettled() # A/project-merge is complete # B/project-merge is complete self.assertBuilds([ dict(name='project-test1', changes='1,1'), dict(name='project-test2', changes='1,1'), dict(name='project-test1', changes='1,1 2,1'), dict(name='project-test2', changes='1,1 2,1'), ]) # Release project-test1 for A which will fail. This will # abort both running B jobs and reexecute project-merge for B. self.builds[0].release() self.waitUntilSettled() self.orderedRelease() self.assertHistory([ dict(name='project-merge', result='SUCCESS', changes='1,1'), dict(name='project-merge', result='SUCCESS', changes='1,1 2,1'), dict(name='project-test1', result='FAILURE', changes='1,1'), dict(name='project-test1', result='ABORTED', changes='1,1 2,1'), dict(name='project-test2', result='ABORTED', changes='1,1 2,1'), dict(name='project-test2', result='SUCCESS', changes='1,1'), dict(name='project-merge', result='SUCCESS', changes='2,1'), dict(name='project-test1', result='SUCCESS', changes='2,1'), dict(name='project-test2', result='SUCCESS', changes='2,1'), ], ordered=False) self.assertEqual(A.data['status'], 'NEW') self.assertEqual(B.data['status'], 'MERGED') self.assertEqual(A.reported, 2) self.assertEqual(B.reported, 2) # Make sure that after a reset of the buildset the files state is # updated correctly and the schedulers is not resolving the list # of changed files via the merger. self.assertIsNone( self.scheds.first.sched.merger.merger_api.history.get( "fileschanges")) def test_independent_queues(self): "Test that changes end up in the right queues" self.executor_server.hold_jobs_in_build = True A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B') C = self.fake_gerrit.addFakeChange('org/project2', 'master', 'C') A.addApproval('Code-Review', 2) B.addApproval('Code-Review', 2) C.addApproval('Code-Review', 2) self.fake_gerrit.addEvent(A.addApproval('Approved', 1)) self.waitUntilSettled() self.fake_gerrit.addEvent(B.addApproval('Approved', 1)) self.waitUntilSettled() self.fake_gerrit.addEvent(C.addApproval('Approved', 1)) self.waitUntilSettled() # There should be one merge job at the head of each queue running self.assertBuilds([ dict(name='project-merge', changes='1,1'), dict(name='project-merge', changes='2,1'), ]) # Release the current merge builds self.builds[0].release() self.waitUntilSettled() self.builds[0].release() self.waitUntilSettled() # Release the merge job for project2 which is behind project1 self.executor_server.release('.*-merge') self.waitUntilSettled() # All the test builds should be running: self.assertBuilds([ dict(name='project-test1', changes='1,1'), dict(name='project-test2', changes='1,1'), dict(name='project-test1', changes='2,1'), dict(name='project-test2', changes='2,1'), dict(name='project1-project2-integration', changes='2,1'), dict(name='project-test1', changes='2,1 3,1'), dict(name='project-test2', changes='2,1 3,1'), dict(name='project1-project2-integration', changes='2,1 3,1'), ]) self.orderedRelease() self.assertHistory([ dict(name='project-merge', result='SUCCESS', changes='1,1'), dict(name='project-merge', result='SUCCESS', changes='2,1'), dict(name='project-merge', result='SUCCESS', changes='2,1 3,1'), dict(name='project-test1', result='SUCCESS', changes='1,1'), dict(name='project-test2', result='SUCCESS', changes='1,1'), dict(name='project-test1', result='SUCCESS', changes='2,1'), dict(name='project-test2', result='SUCCESS', changes='2,1'), dict( name='project1-project2-integration', result='SUCCESS', changes='2,1'), dict(name='project-test1', result='SUCCESS', changes='2,1 3,1'), dict(name='project-test2', result='SUCCESS', changes='2,1 3,1'), dict(name='project1-project2-integration', result='SUCCESS', changes='2,1 3,1'), ]) self.assertEqual(A.data['status'], 'MERGED') self.assertEqual(B.data['status'], 'MERGED') self.assertEqual(C.data['status'], 'MERGED') self.assertEqual(A.reported, 2) self.assertEqual(B.reported, 2) self.assertEqual(C.reported, 2) def test_failed_change_at_head(self): "Test that if a change at the head fails, jobs behind it are canceled" self.executor_server.hold_jobs_in_build = True A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B') C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C') A.addApproval('Code-Review', 2) B.addApproval('Code-Review', 2) C.addApproval('Code-Review', 2) self.executor_server.failJob('project-test1', A) self.fake_gerrit.addEvent(A.addApproval('Approved', 1)) self.fake_gerrit.addEvent(B.addApproval('Approved', 1)) self.fake_gerrit.addEvent(C.addApproval('Approved', 1)) self.waitUntilSettled() self.assertBuilds([ dict(name='project-merge', changes='1,1'), ]) self.executor_server.release('.*-merge') self.waitUntilSettled() self.executor_server.release('.*-merge') self.waitUntilSettled() self.executor_server.release('.*-merge') self.waitUntilSettled() self.assertBuilds([ dict(name='project-test1', changes='1,1'), dict(name='project-test2', changes='1,1'), dict(name='project-test1', changes='1,1 2,1'), dict(name='project-test2', changes='1,1 2,1'), dict(name='project-test1', changes='1,1 2,1 3,1'), dict(name='project-test2', changes='1,1 2,1 3,1'), ]) self.release(self.builds[0]) self.waitUntilSettled() # project-test2, project-merge for B self.assertBuilds([ dict(name='project-test2', changes='1,1'), dict(name='project-merge', changes='2,1'), ]) # Unordered history comparison because the aborts can finish # in any order. self.assertHistory([ dict(name='project-merge', result='SUCCESS', changes='1,1'), dict(name='project-merge', result='SUCCESS', changes='1,1 2,1'), dict(name='project-merge', result='SUCCESS', changes='1,1 2,1 3,1'), dict(name='project-test1', result='FAILURE', changes='1,1'), dict(name='project-test1', result='ABORTED', changes='1,1 2,1'), dict(name='project-test2', result='ABORTED', changes='1,1 2,1'), dict(name='project-test1', result='ABORTED', changes='1,1 2,1 3,1'), dict(name='project-test2', result='ABORTED', changes='1,1 2,1 3,1'), ], ordered=False) self.executor_server.release('.*-merge') self.waitUntilSettled() self.executor_server.release('.*-merge') self.waitUntilSettled() self.orderedRelease() self.assertBuilds([]) self.assertHistory([ dict(name='project-merge', result='SUCCESS', changes='1,1'), dict(name='project-merge', result='SUCCESS', changes='1,1 2,1'), dict(name='project-merge', result='SUCCESS', changes='1,1 2,1 3,1'), dict(name='project-test1', result='FAILURE', changes='1,1'), dict(name='project-test1', result='ABORTED', changes='1,1 2,1'), dict(name='project-test2', result='ABORTED', changes='1,1 2,1'), dict(name='project-test1', result='ABORTED', changes='1,1 2,1 3,1'), dict(name='project-test2', result='ABORTED', changes='1,1 2,1 3,1'), dict(name='project-merge', result='SUCCESS', changes='2,1'), dict(name='project-merge', result='SUCCESS', changes='2,1 3,1'), dict(name='project-test2', result='SUCCESS', changes='1,1'), dict(name='project-test1', result='SUCCESS', changes='2,1'), dict(name='project-test2', result='SUCCESS', changes='2,1'), dict(name='project-test1', result='SUCCESS', changes='2,1 3,1'), dict(name='project-test2', result='SUCCESS', changes='2,1 3,1'), ], ordered=False) self.assertEqual(A.data['status'], 'NEW') self.assertEqual(B.data['status'], 'MERGED') self.assertEqual(C.data['status'], 'MERGED') self.assertEqual(A.reported, 2) self.assertEqual(B.reported, 2) self.assertEqual(C.reported, 2) def test_failed_change_in_middle(self): "Test a failed change in the middle of the queue" self.executor_server.hold_jobs_in_build = True A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B') C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C') A.addApproval('Code-Review', 2) B.addApproval('Code-Review', 2) C.addApproval('Code-Review', 2) self.executor_server.failJob('project-test1', B) self.fake_gerrit.addEvent(A.addApproval('Approved', 1)) self.fake_gerrit.addEvent(B.addApproval('Approved', 1)) self.fake_gerrit.addEvent(C.addApproval('Approved', 1)) self.waitUntilSettled() self.executor_server.release('.*-merge') self.waitUntilSettled() self.executor_server.release('.*-merge') self.waitUntilSettled() self.executor_server.release('.*-merge') self.waitUntilSettled() self.assertEqual(len(self.builds), 6) self.assertEqual(self.builds[0].name, 'project-test1') self.assertEqual(self.builds[1].name, 'project-test2') self.assertEqual(self.builds[2].name, 'project-test1') self.assertEqual(self.builds[3].name, 'project-test2') self.assertEqual(self.builds[4].name, 'project-test1') self.assertEqual(self.builds[5].name, 'project-test2') self.release(self.builds[2]) self.waitUntilSettled() # project-test1 and project-test2 for A # project-test2 for B # project-merge for C (without B) self.assertEqual(len(self.builds), 4) self.assertEqual(self.countJobResults(self.history, 'ABORTED'), 2) self.executor_server.release('.*-merge') self.waitUntilSettled() # project-test1 and project-test2 for A # project-test2 for B # project-test1 and project-test2 for C self.assertEqual(len(self.builds), 5) tenant = self.scheds.first.sched.abide.tenants.get('tenant-one') items = tenant.layout.pipelines['gate'].getAllItems() builds = items[0].current_build_set.getBuilds() self.assertEqual(self.countJobResults(builds, 'SUCCESS'), 1) self.assertEqual(self.countJobResults(builds, None), 2) builds = items[1].current_build_set.getBuilds() self.assertEqual(self.countJobResults(builds, 'SUCCESS'), 1) self.assertEqual(self.countJobResults(builds, 'FAILURE'), 1) self.assertEqual(self.countJobResults(builds, None), 1) builds = items[2].current_build_set.getBuilds() self.assertEqual(self.countJobResults(builds, 'SUCCESS'), 1) self.assertEqual(self.countJobResults(builds, None), 2) self.executor_server.hold_jobs_in_build = False self.executor_server.release() self.waitUntilSettled() self.assertEqual(len(self.builds), 0) self.assertEqual(len(self.history), 12) self.assertEqual(A.data['status'], 'MERGED') self.assertEqual(B.data['status'], 'NEW') self.assertEqual(C.data['status'], 'MERGED') self.assertEqual(A.reported, 2) self.assertEqual(B.reported, 2) self.assertEqual(C.reported, 2) def test_failed_change_at_head_with_queue(self): "Test that if a change at the head fails, queued jobs are canceled" self.hold_jobs_in_queue = True A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B') C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C') A.addApproval('Code-Review', 2) B.addApproval('Code-Review', 2) C.addApproval('Code-Review', 2) self.executor_server.failJob('project-test1', A) self.fake_gerrit.addEvent(A.addApproval('Approved', 1)) self.fake_gerrit.addEvent(B.addApproval('Approved', 1)) self.fake_gerrit.addEvent(C.addApproval('Approved', 1)) self.waitUntilSettled() queue = list(self.executor_api.queued()) self.assertEqual(len(self.builds), 0) self.assertEqual(len(queue), 1) self.assertEqual(queue[0].zone, None) params = self.executor_server.executor_api.getParams(queue[0]) self.assertEqual(params['zuul']['job'], 'project-merge') self.assertEqual(params['items'][0]['number'], '%d' % A.number) self.executor_api.release('.*-merge') self.waitUntilSettled() self.executor_api.release('.*-merge') self.waitUntilSettled() self.executor_api.release('.*-merge') self.waitUntilSettled() queue = list(self.executor_api.queued()) self.assertEqual(len(self.builds), 0) self.assertEqual(len(queue), 6) params = [self.executor_server.executor_api.getParams(x) for x in queue] self.assertEqual(params[0]['zuul']['job'], 'project-test1') self.assertEqual(params[1]['zuul']['job'], 'project-test2') self.assertEqual(params[2]['zuul']['job'], 'project-test1') self.assertEqual(params[3]['zuul']['job'], 'project-test2') self.assertEqual(params[4]['zuul']['job'], 'project-test1') self.assertEqual(params[5]['zuul']['job'], 'project-test2') self.executor_api.release(queue[0]) self.waitUntilSettled() self.assertEqual(len(self.builds), 0) queue = list(self.executor_api.queued()) self.assertEqual(len(queue), 2) # project-test2, project-merge for B self.assertEqual(self.countJobResults(self.history, 'ABORTED'), 0) self.hold_jobs_in_queue = False self.executor_api.release() self.waitUntilSettled() self.assertEqual(len(self.builds), 0) self.assertEqual(len(self.history), 11) self.assertEqual(A.data['status'], 'NEW') self.assertEqual(B.data['status'], 'MERGED') self.assertEqual(C.data['status'], 'MERGED') self.assertEqual(A.reported, 2) self.assertEqual(B.reported, 2) self.assertEqual(C.reported, 2) def _test_time_database(self, iteration): self.executor_server.hold_jobs_in_build = True A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') A.addApproval('Code-Review', 2) self.fake_gerrit.addEvent(A.addApproval('Approved', 1)) self.waitUntilSettled() time.sleep(2) found_job = None pipeline = self.scheds.first.sched.abide.tenants[ 'tenant-one'].layout.pipelines['gate'] pipeline_status = pipeline.formatStatusJSON( self.scheds.first.sched.globals.websocket_url) for queue in pipeline_status['change_queues']: for head in queue['heads']: for item in head: for job in item['jobs']: if job['name'] == 'project-merge': found_job = job break self.assertIsNotNone(found_job) if iteration == 1: self.assertIsNotNone(found_job['estimated_time']) self.assertIsNone(found_job['remaining_time']) else: self.assertIsNotNone(found_job['estimated_time']) self.assertTrue(found_job['estimated_time'] >= 2) self.assertIsNotNone(found_job['remaining_time']) self.executor_server.hold_jobs_in_build = False self.executor_server.release() self.waitUntilSettled() def test_time_database(self): "Test the time database" self._test_time_database(1) self._test_time_database(2) def test_two_failed_changes_at_head(self): "Test that changes are reparented correctly if 2 fail at head" self.executor_server.hold_jobs_in_build = True A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B') C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C') A.addApproval('Code-Review', 2) B.addApproval('Code-Review', 2) C.addApproval('Code-Review', 2) self.executor_server.failJob('project-test1', A) self.executor_server.failJob('project-test1', B) self.fake_gerrit.addEvent(A.addApproval('Approved', 1)) self.fake_gerrit.addEvent(B.addApproval('Approved', 1)) self.fake_gerrit.addEvent(C.addApproval('Approved', 1)) self.waitUntilSettled() self.executor_server.release('.*-merge') self.waitUntilSettled() self.executor_server.release('.*-merge') self.waitUntilSettled() self.executor_server.release('.*-merge') self.waitUntilSettled() self.assertEqual(len(self.builds), 6) self.assertEqual(self.builds[0].name, 'project-test1') self.assertEqual(self.builds[1].name, 'project-test2') self.assertEqual(self.builds[2].name, 'project-test1') self.assertEqual(self.builds[3].name, 'project-test2') self.assertEqual(self.builds[4].name, 'project-test1') self.assertEqual(self.builds[5].name, 'project-test2') self.assertTrue(self.builds[0].hasChanges(A)) self.assertTrue(self.builds[2].hasChanges(A)) self.assertTrue(self.builds[2].hasChanges(B)) self.assertTrue(self.builds[4].hasChanges(A)) self.assertTrue(self.builds[4].hasChanges(B)) self.assertTrue(self.builds[4].hasChanges(C)) # Fail change B first self.release(self.builds[2]) self.waitUntilSettled() # restart of C after B failure self.executor_server.release('.*-merge') self.waitUntilSettled() self.assertEqual(len(self.builds), 5) self.assertEqual(self.builds[0].name, 'project-test1') self.assertEqual(self.builds[1].name, 'project-test2') self.assertEqual(self.builds[2].name, 'project-test2') self.assertEqual(self.builds[3].name, 'project-test1') self.assertEqual(self.builds[4].name, 'project-test2') self.assertTrue(self.builds[1].hasChanges(A)) self.assertTrue(self.builds[2].hasChanges(A)) self.assertTrue(self.builds[2].hasChanges(B)) self.assertTrue(self.builds[4].hasChanges(A)) self.assertFalse(self.builds[4].hasChanges(B)) self.assertTrue(self.builds[4].hasChanges(C)) # Finish running all passing jobs for change A self.release(self.builds[1]) self.waitUntilSettled() # Fail and report change A self.release(self.builds[0]) self.waitUntilSettled() # restart of B,C after A failure self.executor_server.release('.*-merge') self.waitUntilSettled() self.executor_server.release('.*-merge') self.waitUntilSettled() self.assertEqual(len(self.builds), 4) self.assertEqual(self.builds[0].name, 'project-test1') # B self.assertEqual(self.builds[1].name, 'project-test2') # B self.assertEqual(self.builds[2].name, 'project-test1') # C self.assertEqual(self.builds[3].name, 'project-test2') # C self.assertFalse(self.builds[1].hasChanges(A)) self.assertTrue(self.builds[1].hasChanges(B)) self.assertFalse(self.builds[1].hasChanges(C)) self.assertFalse(self.builds[2].hasChanges(A)) # After A failed and B and C restarted, B should be back in # C's tests because it has not failed yet. self.assertTrue(self.builds[2].hasChanges(B)) self.assertTrue(self.builds[2].hasChanges(C)) self.executor_server.hold_jobs_in_build = False self.executor_server.release() self.waitUntilSettled() self.assertEqual(len(self.builds), 0) self.assertEqual(len(self.history), 21) self.assertEqual(A.data['status'], 'NEW') self.assertEqual(B.data['status'], 'NEW') self.assertEqual(C.data['status'], 'MERGED') self.assertEqual(A.reported, 2) self.assertEqual(B.reported, 2) self.assertEqual(C.reported, 2) def test_patch_order(self): "Test that dependent patches are tested in the right order" A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B') C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C') A.addApproval('Code-Review', 2) B.addApproval('Code-Review', 2) C.addApproval('Code-Review', 2) M2 = self.fake_gerrit.addFakeChange('org/project', 'master', 'M2') M1 = self.fake_gerrit.addFakeChange('org/project', 'master', 'M1') M2.setMerged() M1.setMerged() # C -> B -> A -> M1 -> M2 # M2 is here to make sure it is never queried. If it is, it # means zuul is walking down the entire history of merged # changes. C.setDependsOn(B, 1) B.setDependsOn(A, 1) A.setDependsOn(M1, 1) M1.setDependsOn(M2, 1) self.fake_gerrit.addEvent(C.addApproval('Approved', 1)) self.waitUntilSettled() self.assertEqual(A.data['status'], 'NEW') self.assertEqual(B.data['status'], 'NEW') self.assertEqual(C.data['status'], 'NEW') self.fake_gerrit.addEvent(B.addApproval('Approved', 1)) self.fake_gerrit.addEvent(A.addApproval('Approved', 1)) self.waitUntilSettled() self.assertEqual(M2.queried, 0) self.assertEqual(A.data['status'], 'MERGED') self.assertEqual(B.data['status'], 'MERGED') self.assertEqual(C.data['status'], 'MERGED') self.assertEqual(A.reported, 2) self.assertEqual(B.reported, 2) self.assertEqual(C.reported, 2) def test_needed_changes_enqueue(self): "Test that a needed change is enqueued ahead" # A Given a git tree like this, if we enqueue # / \ change C, we should walk up and down the tree # B G and enqueue changes in the order ABCDEFG. # /|\ This is also the order that you would get if # *C E F you enqueued changes in the order ABCDEFG, so # / the ordering is stable across re-enqueue events. # D A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B') C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C') D = self.fake_gerrit.addFakeChange('org/project', 'master', 'D') E = self.fake_gerrit.addFakeChange('org/project', 'master', 'E') F = self.fake_gerrit.addFakeChange('org/project', 'master', 'F') G = self.fake_gerrit.addFakeChange('org/project', 'master', 'G') B.setDependsOn(A, 1) C.setDependsOn(B, 1) D.setDependsOn(C, 1) E.setDependsOn(B, 1) F.setDependsOn(B, 1) G.setDependsOn(A, 1) A.addApproval('Code-Review', 2) B.addApproval('Code-Review', 2) C.addApproval('Code-Review', 2) D.addApproval('Code-Review', 2) E.addApproval('Code-Review', 2) F.addApproval('Code-Review', 2) G.addApproval('Code-Review', 2) self.fake_gerrit.addEvent(C.addApproval('Approved', 1)) self.waitUntilSettled() self.assertEqual(A.data['status'], 'NEW') self.assertEqual(B.data['status'], 'NEW') self.assertEqual(C.data['status'], 'NEW') self.assertEqual(D.data['status'], 'NEW') self.assertEqual(E.data['status'], 'NEW') self.assertEqual(F.data['status'], 'NEW') self.assertEqual(G.data['status'], 'NEW') # We're about to add approvals to changes without adding the # triggering events to Zuul, so that we can be sure that it is # enqueing the changes based on dependencies, not because of # triggering events. Since it will have the changes cached # already (without approvals), we need to clear the cache # first. for connection in self.scheds.first.connections.connections.values(): if hasattr(connection, '_change_cache'): connection.maintainCache([], max_age=0) self.executor_server.hold_jobs_in_build = True A.addApproval('Approved', 1) B.addApproval('Approved', 1) D.addApproval('Approved', 1) E.addApproval('Approved', 1) F.addApproval('Approved', 1) G.addApproval('Approved', 1) self.fake_gerrit.addEvent(C.addApproval('Approved', 1)) for x in range(8): self.executor_server.release('.*-merge') self.waitUntilSettled() self.executor_server.hold_jobs_in_build = False self.executor_server.release() self.waitUntilSettled() self.assertEqual(A.data['status'], 'MERGED') self.assertEqual(B.data['status'], 'MERGED') self.assertEqual(C.data['status'], 'MERGED') self.assertEqual(D.data['status'], 'MERGED') self.assertEqual(E.data['status'], 'MERGED') self.assertEqual(F.data['status'], 'MERGED') self.assertEqual(G.data['status'], 'MERGED') self.assertEqual(A.reported, 2) self.assertEqual(B.reported, 2) self.assertEqual(C.reported, 2) self.assertEqual(D.reported, 2) self.assertEqual(E.reported, 2) self.assertEqual(F.reported, 2) self.assertEqual(G.reported, 2) self.assertEqual(self.history[6].changes, '1,1 2,1 3,1 4,1 5,1 6,1 7,1') def test_source_cache(self): "Test that the source cache operates correctly" self.executor_server.hold_jobs_in_build = True A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B') X = self.fake_gerrit.addFakeChange('org/project', 'master', 'X') A.addApproval('Code-Review', 2) B.addApproval('Code-Review', 2) M1 = self.fake_gerrit.addFakeChange('org/project', 'master', 'M1') M1.setMerged() B.setDependsOn(A, 1) A.setDependsOn(M1, 1) self.fake_gerrit.addEvent(A.addApproval('Approved', 1)) self.fake_gerrit.addEvent(X.getPatchsetCreatedEvent(1)) self.waitUntilSettled() for build in self.builds: if build.pipeline == 'check': build.release() self.waitUntilSettled() for build in self.builds: if build.pipeline == 'check': build.release() self.waitUntilSettled() self.fake_gerrit.addEvent(B.addApproval('Approved', 1)) self.waitUntilSettled() cached_changes = list(self.fake_gerrit._change_cache) self.log.debug("len %s", [c.cache_key for c in cached_changes]) # there should still be changes in the cache self.assertNotEqual(len(cached_changes), 0) self.executor_server.hold_jobs_in_build = False self.executor_server.release() self.waitUntilSettled() self.assertEqual(A.data['status'], 'MERGED') self.assertEqual(B.data['status'], 'MERGED') self.assertEqual(A.queried, 2) # Initial and isMerged self.assertEqual(B.queried, 3) # Initial A, refresh from B, isMerged def test_connection_cache_cleanup(self): "Test that cached changes are correctly cleaned up" sched = self.scheds.first.sched def _getCachedChanges(): cached = set() for source in sched.connections.getSources(): cached.update(source.getCachedChanges()) return cached A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B') B.setDependsOn(A, 1) self.hold_jobs_in_queue = True self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) self.waitUntilSettled() C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C') C.setMerged() self.fake_gerrit.addEvent(C.getRefUpdatedEvent()) self.waitUntilSettled() self.assertEqual(len(_getCachedChanges()), 3) sched.maintainConnectionCache() self.assertEqual(len(_getCachedChanges()), 3) # Test this method separately to make sure we are getting # cache keys of the correct type, since we don't do run-time # validation. relevant = sched._gatherConnectionCacheKeys() self.assertEqual(len(relevant), 2) for k in relevant: if not isinstance(k, ChangeKey): raise RuntimeError("Cache key %s is not a ChangeKey" % repr(k)) self.hold_jobs_in_queue = False self.executor_api.release() self.waitUntilSettled() self.assertEqual(len(_getCachedChanges()), 3) sched.maintainConnectionCache() self.assertEqual(len(_getCachedChanges()), 3) # Test that outdated but still relevant changes are not cleaned up for connection in sched.connections.connections.values(): connection.maintainCache( set([c.cache_stat.key for c in _getCachedChanges()]), max_age=0) self.assertEqual(len(_getCachedChanges()), 3) change1 = None change2 = None for c in _getCachedChanges(): if c.cache_stat.key.stable_id == '1': change1 = c if c.cache_stat.key.stable_id == '2': change2 = c # Make change1 eligible for cleanup, but not change2 change1.cache_stat = zuul.model.CacheStat(change1.cache_stat.key, change1.cache_stat.uuid, change1.cache_stat.version, change1.cache_stat.mzxid, 0.0, 0, 0) # We should not delete change1 since it's needed by change2 # which we want to keep. for connection in sched.connections.connections.values(): connection.maintainCache([], max_age=7200) self.assertEqual(len(_getCachedChanges()), 3) # Make both changes eligible for deletion change2.cache_stat = zuul.model.CacheStat(change2.cache_stat.key, change2.cache_stat.uuid, change2.cache_stat.version, change1.cache_stat.mzxid, 0.0, 0, 0) for connection in sched.connections.connections.values(): connection.maintainCache([], max_age=7200) # The master branch change remains self.assertEqual(len(_getCachedChanges()), 1) # Test that we can remove changes once max_age has expired for connection in sched.connections.connections.values(): connection.maintainCache([], max_age=0) self.assertEqual(len(_getCachedChanges()), 0) def test_can_merge(self): "Test whether a change is ready to merge" # TODO: move to test_gerrit (this is a unit test!) A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') tenant = self.scheds.first.sched.abide.tenants.get('tenant-one') (trusted, project) = tenant.getProject('org/project') source = project.source # TODO(pabelanger): As we add more source / trigger APIs we should make # it easier for users to create events for testing. event = zuul.model.TriggerEvent() event.trigger_name = 'gerrit' event.change_number = '1' event.patch_number = '2' a = source.getChange(source.getChangeKey(event), event=event) mgr = tenant.layout.pipelines['gate'].manager self.assertFalse(source.canMerge(a, mgr.getSubmitAllowNeeds())) A.addApproval('Code-Review', 2) a = source.getChange(source.getChangeKey(event), refresh=True, event=event) self.assertFalse(source.canMerge(a, mgr.getSubmitAllowNeeds())) A.addApproval('Approved', 1) a = source.getChange(source.getChangeKey(event), refresh=True, event=event) self.assertTrue(source.canMerge(a, mgr.getSubmitAllowNeeds())) A.setWorkInProgress(True) a = source.getChange(source.getChangeKey(event), refresh=True, event=event) self.assertFalse(source.canMerge(a, mgr.getSubmitAllowNeeds())) def test_project_merge_conflict(self): "Test that gate merge conflicts are handled properly" self.hold_jobs_in_queue = True A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A', files={'conflict': 'foo'}) B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B', files={'conflict': 'bar'}) C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C') A.addApproval('Code-Review', 2) B.addApproval('Code-Review', 2) C.addApproval('Code-Review', 2) self.fake_gerrit.addEvent(A.addApproval('Approved', 1)) self.fake_gerrit.addEvent(B.addApproval('Approved', 1)) self.fake_gerrit.addEvent(C.addApproval('Approved', 1)) self.waitUntilSettled() self.assertEqual(A.reported, 1) self.assertEqual(C.reported, 1) self.executor_api.release('project-merge') self.waitUntilSettled() self.executor_api.release('project-merge') self.waitUntilSettled() self.executor_api.release('project-merge') self.waitUntilSettled() self.hold_jobs_in_queue = False self.executor_api.release() self.waitUntilSettled() self.assertEqual(A.data['status'], 'MERGED') self.assertEqual(B.data['status'], 'NEW') self.assertEqual(C.data['status'], 'MERGED') self.assertEqual(A.reported, 2) self.assertIn('Merge Failed', B.messages[-1]) self.assertEqual(C.reported, 2) self.assertHistory([ dict(name='project-merge', result='SUCCESS', changes='1,1'), dict(name='project-test1', result='SUCCESS', changes='1,1'), dict(name='project-test2', result='SUCCESS', changes='1,1'), dict(name='project-merge', result='SUCCESS', changes='1,1 3,1'), dict(name='project-test1', result='SUCCESS', changes='1,1 3,1'), dict(name='project-test2', result='SUCCESS', changes='1,1 3,1'), ], ordered=False) def test_delayed_merge_conflict(self): "Test that delayed check merge conflicts are handled properly" # Hold jobs in the ZooKeeper queue so that we can test whether # the executor sucesfully merges a change based on an old # repo state (frozen by the scheduler) which would otherwise # conflict. self.hold_jobs_in_queue = True A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A', files={'conflict': 'foo'}) B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B', files={'conflict': 'bar'}) C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C') C.setDependsOn(B, 1) # A enters the gate queue; B and C enter the check queue A.addApproval('Code-Review', 2) self.fake_gerrit.addEvent(A.addApproval('Approved', 1)) self.waitUntilSettled() self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1)) self.waitUntilSettled() self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1)) self.waitUntilSettled() self.assertEqual(A.reported, 1) self.assertEqual(B.reported, 0) # Check does not report start self.assertEqual(C.reported, 0) # Check does not report start # A merges while B and C are queued in check # Release A project-merge queue = list(self.executor_api.queued()) self.executor_api.release(queue[0]) self.waitUntilSettled() # Release A project-test* # gate has higher precedence, so A's test jobs are added in # front of the merge jobs for B and C queue = list(self.executor_api.queued()) self.executor_api.release(queue[0]) self.executor_api.release(queue[1]) self.waitUntilSettled() self.assertEqual(A.data['status'], 'MERGED') self.assertEqual(B.data['status'], 'NEW') self.assertEqual(C.data['status'], 'NEW') self.assertEqual(A.reported, 2) self.assertEqual(B.reported, 0) self.assertEqual(C.reported, 0) self.assertHistory([ dict(name='project-merge', result='SUCCESS', changes='1,1'), dict(name='project-test1', result='SUCCESS', changes='1,1'), dict(name='project-test2', result='SUCCESS', changes='1,1'), ], ordered=False) # B and C report merge conflicts # Release B project-merge queue = list(self.executor_api.queued()) self.executor_api.release(queue[0]) self.waitUntilSettled() # Release C self.hold_jobs_in_queue = False self.executor_api.release() self.waitUntilSettled() self.assertEqual(A.data['status'], 'MERGED') self.assertEqual(B.data['status'], 'NEW') self.assertEqual(C.data['status'], 'NEW') self.assertEqual(A.reported, 2) self.assertEqual(B.reported, 1) self.assertEqual(C.reported, 1) self.assertHistory([ dict(name='project-merge', result='SUCCESS', changes='1,1'), dict(name='project-test1', result='SUCCESS', changes='1,1'), dict(name='project-test2', result='SUCCESS', changes='1,1'), dict(name='project-merge', result='SUCCESS', changes='2,1'), dict(name='project-test1', result='SUCCESS', changes='2,1'), dict(name='project-test2', result='SUCCESS', changes='2,1'), dict(name='project-merge', result='SUCCESS', changes='2,1 3,1'), dict(name='project-test1', result='SUCCESS', changes='2,1 3,1'), dict(name='project-test2', result='SUCCESS', changes='2,1 3,1'), ], ordered=False) def test_post(self): "Test that post jobs run" p = "review.example.com/org/project" upstream = self.getUpstreamRepos([p]) A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') A.setMerged() A_commit = str(upstream[p].commit('master')) self.log.debug("A commit: %s" % A_commit) e = { "type": "ref-updated", "submitter": { "name": "User Name", }, "refUpdate": { "oldRev": "90f173846e3af9154517b88543ffbd1691f31366", "newRev": A_commit, "refName": "master", "project": "org/project", } } self.fake_gerrit.addEvent(e) self.waitUntilSettled() job_names = [x.name for x in self.history] self.assertEqual(len(self.history), 1) self.assertIn('project-post', job_names) @simple_layout('layouts/negate-post.yaml') def test_post_negative_regex(self): "Test that post jobs run" p = "review.example.com/org/project" upstream = self.getUpstreamRepos([p]) A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') A.setMerged() A_commit = str(upstream[p].commit('master')) self.log.debug("A commit: %s" % A_commit) e = { "type": "ref-updated", "submitter": { "name": "User Name", }, "refUpdate": { "oldRev": "90f173846e3af9154517b88543ffbd1691f31366", "newRev": A_commit, "refName": "master", "project": "org/project", } } self.fake_gerrit.addEvent(e) self.waitUntilSettled() job_names = [x.name for x in self.history] self.assertEqual(len(self.history), 1) self.assertIn('post-job', job_names) def test_post_ignore_deletes(self): "Test that deleting refs does not trigger post jobs" e = { "type": "ref-updated", "submitter": { "name": "User Name", }, "refUpdate": { "oldRev": "90f173846e3af9154517b88543ffbd1691f31366", "newRev": "0000000000000000000000000000000000000000", "refName": "master", "project": "org/project", } } self.fake_gerrit.addEvent(e) self.waitUntilSettled() job_names = [x.name for x in self.history] self.assertEqual(len(self.history), 0) self.assertNotIn('project-post', job_names) @simple_layout('layouts/dont-ignore-ref-deletes.yaml') def test_post_ignore_deletes_negative(self): "Test that deleting refs does trigger post jobs" e = { "type": "ref-updated", "submitter": { "name": "User Name", }, "refUpdate": { "oldRev": "90f173846e3af9154517b88543ffbd1691f31366", "newRev": "0000000000000000000000000000000000000000", "refName": "testbranch", "project": "org/project", } } self.fake_gerrit.addEvent(e) self.waitUntilSettled() job_names = [x.name for x in self.history] self.assertEqual(len(self.history), 1) self.assertIn('project-post', job_names) @skip("Disabled for early v3 development") def test_build_configuration_branch_interaction(self): "Test that switching between branches works" self.test_build_configuration() self.test_build_configuration_branch() # C has been merged, undo that path = os.path.join(self.upstream_root, "org/project") repo = git.Repo(path) repo.heads.master.commit = repo.commit('init') self.test_build_configuration() def test_dependent_changes_rebase(self): # Test that no errors occur when we walk a dependency tree # with an unused leaf node due to a rebase. # Start by constructing: C -> B -> A A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B') B.setDependsOn(A, 1) C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C') C.setDependsOn(B, 1) # Then rebase to form: D -> C -> A C.addPatchset() # C,2 C.setDependsOn(A, 1) D = self.fake_gerrit.addFakeChange('org/project', 'master', 'D') D.setDependsOn(C, 2) # Walk the entire tree self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) self.waitUntilSettled() self.assertEqual(len(self.history), 3) # Verify that walking just part of the tree still works self.fake_gerrit.addEvent(D.getPatchsetCreatedEvent(1)) self.waitUntilSettled() self.assertEqual(len(self.history), 6) def test_dependent_changes_dequeue(self): "Test that dependent patches are not needlessly tested" A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B') C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C') A.addApproval('Code-Review', 2) B.addApproval('Code-Review', 2) C.addApproval('Code-Review', 2) M1 = self.fake_gerrit.addFakeChange('org/project', 'master', 'M1') M1.setMerged() # C -> B -> A -> M1 C.setDependsOn(B, 1) B.setDependsOn(A, 1) A.setDependsOn(M1, 1) self.executor_server.failJob('project-merge', A) self.fake_gerrit.addEvent(C.addApproval('Approved', 1)) self.fake_gerrit.addEvent(B.addApproval('Approved', 1)) self.fake_gerrit.addEvent(A.addApproval('Approved', 1)) self.waitUntilSettled() self.assertEqual(A.data['status'], 'NEW') self.assertEqual(A.reported, 2) self.assertEqual(B.data['status'], 'NEW') self.assertEqual(B.reported, 2) self.assertEqual(C.data['status'], 'NEW') self.assertIn('This change depends on a change that failed to merge.', C.messages[-1]) self.assertEqual(len(self.history), 1) def test_failing_dependent_changes(self): "Test that failing dependent patches are taken out of stream" self.executor_server.hold_jobs_in_build = True A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B') C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C') D = self.fake_gerrit.addFakeChange('org/project', 'master', 'D') E = self.fake_gerrit.addFakeChange('org/project', 'master', 'E') A.addApproval('Code-Review', 2) B.addApproval('Code-Review', 2) C.addApproval('Code-Review', 2) D.addApproval('Code-Review', 2) E.addApproval('Code-Review', 2) # E, D -> C -> B, A D.setDependsOn(C, 1) C.setDependsOn(B, 1) self.executor_server.failJob('project-test1', B) self.fake_gerrit.addEvent(A.addApproval('Approved', 1)) self.fake_gerrit.addEvent(D.addApproval('Approved', 1)) self.fake_gerrit.addEvent(C.addApproval('Approved', 1)) self.fake_gerrit.addEvent(B.addApproval('Approved', 1)) self.fake_gerrit.addEvent(E.addApproval('Approved', 1)) self.waitUntilSettled() self.executor_server.release('.*-merge') self.waitUntilSettled() self.executor_server.release('.*-merge') self.waitUntilSettled() self.executor_server.release('.*-merge') self.waitUntilSettled() self.executor_server.release('.*-merge') self.waitUntilSettled() self.executor_server.release('.*-merge') self.waitUntilSettled() self.executor_server.hold_jobs_in_build = False for build in self.builds: if build.parameters['zuul']['change'] != '1': build.release() self.waitUntilSettled() self.executor_server.release() self.waitUntilSettled() self.assertEqual(A.data['status'], 'MERGED') self.assertEqual(A.reported, 2) self.assertIn('Build succeeded', A.messages[1]) self.assertEqual(B.data['status'], 'NEW') self.assertEqual(B.reported, 2) self.assertIn('Build failed', B.messages[1]) self.assertEqual(C.data['status'], 'NEW') self.assertEqual(C.reported, 2) self.assertIn('depends on a change', C.messages[1]) self.assertEqual(D.data['status'], 'NEW') self.assertEqual(D.reported, 2) self.assertIn('depends on a change', D.messages[1]) self.assertEqual(E.data['status'], 'MERGED') self.assertEqual(E.reported, 2) self.assertIn('Build succeeded', E.messages[1]) self.assertEqual(len(self.history), 18) def test_head_is_dequeued_once(self): "Test that if a change at the head fails it is dequeued only once" # If it's dequeued more than once, we should see extra # aborted jobs. self.executor_server.hold_jobs_in_build = True A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B') C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C') A.addApproval('Code-Review', 2) B.addApproval('Code-Review', 2) C.addApproval('Code-Review', 2) self.executor_server.failJob('project-test1', A) self.executor_server.failJob('project-test2', A) self.fake_gerrit.addEvent(A.addApproval('Approved', 1)) self.fake_gerrit.addEvent(B.addApproval('Approved', 1)) self.fake_gerrit.addEvent(C.addApproval('Approved', 1)) self.waitUntilSettled() self.assertEqual(len(self.builds), 1) self.assertEqual(self.builds[0].name, 'project-merge') self.assertTrue(self.builds[0].hasChanges(A)) self.executor_server.release('.*-merge') self.waitUntilSettled() self.executor_server.release('.*-merge') self.waitUntilSettled() self.executor_server.release('.*-merge') self.waitUntilSettled() self.assertEqual(len(self.builds), 6) self.assertEqual(self.builds[0].name, 'project-test1') self.assertEqual(self.builds[1].name, 'project-test2') self.assertEqual(self.builds[2].name, 'project-test1') self.assertEqual(self.builds[3].name, 'project-test2') self.assertEqual(self.builds[4].name, 'project-test1') self.assertEqual(self.builds[5].name, 'project-test2') self.release(self.builds[0]) self.waitUntilSettled() self.assertEqual(len(self.builds), 2) # test2, merge for B self.assertEqual(self.countJobResults(self.history, 'ABORTED'), 4) self.executor_server.hold_jobs_in_build = False self.executor_server.release() self.waitUntilSettled() self.assertEqual(len(self.builds), 0) self.assertEqual(len(self.history), 15) self.assertEqual(A.data['status'], 'NEW') self.assertEqual(B.data['status'], 'MERGED') self.assertEqual(C.data['status'], 'MERGED') self.assertEqual(A.reported, 2) self.assertEqual(B.reported, 2) self.assertEqual(C.reported, 2) def test_approval_removal(self): # Test that we dequeue a change when it can not merge self.executor_server.hold_jobs_in_build = True A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') A.addApproval('Code-Review', 2) self.fake_gerrit.addEvent(A.addApproval('Approved', 1)) self.waitUntilSettled() self.assertEqual(1, len(self.builds)) self.assertEqual(0, len(self.history)) # Remove the approval self.fake_gerrit.addEvent(A.addApproval('Approved', 0)) self.waitUntilSettled() self.executor_server.hold_jobs_in_build = False self.executor_server.release() self.waitUntilSettled() # The change should be dequeued. self.assertHistory([ dict(name='project-merge', result='ABORTED'), ], ordered=False) self.assertEqual(2, len(A.messages)) self.assertEqual(A.data['status'], 'NEW') self.assertIn('This change is unable to merge ' 'due to a missing merge requirement.', A.messages[1]) @simple_layout('layouts/nonvoting-job-approval.yaml') def test_nonvoting_job_approval(self): "Test that non-voting jobs don't vote but leave approval" A = self.fake_gerrit.addFakeChange('org/nonvoting-project', 'master', 'A') self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) self.executor_server.failJob('nonvoting-project-test2', A) self.waitUntilSettled() self.assertEqual(A.data['status'], 'NEW') self.assertEqual(A.reported, 1) self.assertEqual( self.getJobFromHistory('nonvoting-project-test1').result, 'SUCCESS') self.assertEqual( self.getJobFromHistory('nonvoting-project-test2').result, 'FAILURE') self.assertFalse(self.getJobFromHistory('nonvoting-project-test1'). parameters['zuul']['voting']) self.assertFalse(self.getJobFromHistory('nonvoting-project-test2'). parameters['zuul']['voting']) self.assertEqual(A.patchsets[0]['approvals'][0]['value'], "1") @simple_layout('layouts/nonvoting-job.yaml') def test_nonvoting_job(self): "Test that non-voting jobs don't vote." A = self.fake_gerrit.addFakeChange('org/nonvoting-project', 'master', 'A') A.addApproval('Code-Review', 2) self.executor_server.failJob('nonvoting-project-test2', A) self.fake_gerrit.addEvent(A.addApproval('Approved', 1)) self.waitUntilSettled() self.assertEqual(A.data['status'], 'MERGED') self.assertEqual(A.reported, 2) self.assertEqual( self.getJobFromHistory('nonvoting-project-merge').result, 'SUCCESS') self.assertEqual( self.getJobFromHistory('nonvoting-project-test1').result, 'SUCCESS') self.assertEqual( self.getJobFromHistory('nonvoting-project-test2').result, 'FAILURE') self.assertTrue(self.getJobFromHistory('nonvoting-project-merge'). parameters['zuul']['voting']) self.assertTrue(self.getJobFromHistory('nonvoting-project-test1'). parameters['zuul']['voting']) self.assertFalse(self.getJobFromHistory('nonvoting-project-test2'). parameters['zuul']['voting']) def test_check_queue_success(self): "Test successful check queue jobs." A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) self.waitUntilSettled() self.assertEqual(A.data['status'], 'NEW') self.assertEqual(A.reported, 1) self.assertEqual(self.getJobFromHistory('project-merge').result, 'SUCCESS') self.assertEqual(self.getJobFromHistory('project-test1').result, 'SUCCESS') self.assertEqual(self.getJobFromHistory('project-test2').result, 'SUCCESS') def test_check_queue_failure(self): "Test failed check queue jobs." A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') self.executor_server.failJob('project-test2', A) self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) self.waitUntilSettled() self.assertEqual(A.data['status'], 'NEW') self.assertEqual(A.reported, 1) self.assertEqual(self.getJobFromHistory('project-merge').result, 'SUCCESS') self.assertEqual(self.getJobFromHistory('project-test1').result, 'SUCCESS') self.assertEqual(self.getJobFromHistory('project-test2').result, 'FAILURE') @simple_layout('layouts/autohold.yaml') def test_autohold(self): self.scheds.first.sched.autohold( 'tenant-one', 'review.example.com/org/project', 'project-test2', ".*", "reason text", 1, None) # There should be a record in ZooKeeper request_list = self.sched_zk_nodepool.getHoldRequests() self.assertEqual(1, len(request_list)) request = self.sched_zk_nodepool.getHoldRequest( request_list[0]) self.assertIsNotNone(request) self.assertEqual('tenant-one', request.tenant) self.assertEqual('review.example.com/org/project', request.project) self.assertEqual('project-test2', request.job) self.assertEqual('reason text', request.reason) self.assertEqual(1, request.max_count) self.assertEqual(0, request.current_count) self.assertEqual([], request.nodes) # Some convenience variables for checking the stats. tenant_ram_stat =\ 'zuul.nodepool.resources.in_use.tenant.tenant-one.ram' project_ram_stat = ('zuul.nodepool.resources.in_use.project.' 'review_example_com/org/project.ram') # Test that we zeroed the gauges self.scheds.first.sched._runStats() self.assertUnReportedStat(tenant_ram_stat, value='1024', kind='g') self.assertUnReportedStat(project_ram_stat, value='1024', kind='g') self.assertReportedStat(tenant_ram_stat, value='0', kind='g') self.assertReportedStat(project_ram_stat, value='0', kind='g') # First check that successful jobs do not autohold A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) self.waitUntilSettled() self.assertEqual(A.data['status'], 'NEW') self.assertEqual(A.reported, 1) # project-test2 self.assertEqual(self.history[0].result, 'SUCCESS') # Check nodepool for a held node held_node = None for node in self.fake_nodepool.getNodes(): if node['state'] == zuul.model.STATE_HOLD: held_node = node break self.assertIsNone(held_node) # Hold in build to check the stats self.executor_server.hold_jobs_in_build = True # Now test that failed jobs are autoheld # Set resources only for this node so we can examine the code # path for updating the stats on autohold. self.fake_nodepool.resources = { 'cores': 2, 'ram': 1024, 'instances': 1, } B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B') self.executor_server.failJob('project-test2', B) self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1)) self.waitUntilSettled() # Get the build request object build = list(self.getCurrentBuilds())[0] # We should report using the held node's resources self.waitUntilNodeCacheSync( self.scheds.first.sched.nodepool.zk_nodepool) self.statsd.clear() self.scheds.first.sched._runStats() self.assertReportedStat(tenant_ram_stat, value='1024', kind='g') self.assertReportedStat(project_ram_stat, value='1024', kind='g') self.assertUnReportedStat(tenant_ram_stat, value='0', kind='g') self.assertUnReportedStat(project_ram_stat, value='0', kind='g') self.executor_server.hold_jobs_in_build = False self.executor_server.release() self.waitUntilSettled() self.assertEqual(B.data['status'], 'NEW') self.assertEqual(B.reported, 1) # project-test2 self.assertEqual(self.history[1].result, 'FAILURE') self.assertTrue(build.held) # Check nodepool for a held node held_node = None for node in self.fake_nodepool.getNodes(): if node['state'] == zuul.model.STATE_HOLD: held_node = node break self.assertIsNotNone(held_node) # Validate node has recorded the failed job self.assertEqual( held_node['hold_job'], " ".join(['tenant-one', 'review.example.com/org/project', 'project-test2', '.*']) ) self.assertEqual(held_node['comment'], "reason text") # The hold request current_count should have incremented # and we should have recorded the held node ID. request2 = self.sched_zk_nodepool.getHoldRequest( request.id) self.assertEqual(request.current_count + 1, request2.current_count) self.assertEqual(1, len(request2.nodes)) self.assertEqual(1, len(request2.nodes[0]["nodes"])) # We should still report we use the resources self.waitUntilNodeCacheSync( self.scheds.first.sched.nodepool.zk_nodepool) self.statsd.clear() self.scheds.first.sched._runStats() self.assertReportedStat(tenant_ram_stat, value='1024', kind='g') self.assertReportedStat(project_ram_stat, value='1024', kind='g') self.assertUnReportedStat(tenant_ram_stat, value='0', kind='g') self.assertUnReportedStat(project_ram_stat, value='0', kind='g') # Another failed change should not hold any more nodes self.fake_nodepool.resources = {} C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C') self.executor_server.failJob('project-test2', C) self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1)) self.waitUntilSettled() self.assertEqual(C.data['status'], 'NEW') self.assertEqual(C.reported, 1) # project-test2 self.assertEqual(self.history[2].result, 'FAILURE') held_nodes = 0 for node in self.fake_nodepool.getNodes(): if node['state'] == zuul.model.STATE_HOLD: held_nodes += 1 self.assertEqual(held_nodes, 1) # request current_count should not have changed request3 = self.sched_zk_nodepool.getHoldRequest( request2.id) self.assertEqual(request2.current_count, request3.current_count) # Deleting hold request should set held nodes to used self.sched_zk_nodepool.deleteHoldRequest(request3) node_states = [n['state'] for n in self.fake_nodepool.getNodes()] self.assertEqual(3, len(node_states)) self.assertEqual([zuul.model.STATE_USED] * 3, node_states) # Nodepool deletes the nodes for n in self.fake_nodepool.getNodes(): self.fake_nodepool.removeNode(n) # We should now report that we no longer use the nodes resources self.waitUntilNodeCacheSync( self.scheds.first.sched.nodepool.zk_nodepool) self.statsd.clear() self.scheds.first.sched._runStats() self.assertUnReportedStat(tenant_ram_stat, value='1024', kind='g') self.assertUnReportedStat(project_ram_stat, value='1024', kind='g') self.assertReportedStat(tenant_ram_stat, value='0', kind='g') self.assertReportedStat(project_ram_stat, value='0', kind='g') @simple_layout('layouts/autohold.yaml') def test_autohold_info(self): self.scheds.first.sched.autohold( 'tenant-one', 'review.example.com/org/project', 'project-test2', ".*", "reason text", 1, None) # There should be a record in ZooKeeper request_list = self.sched_zk_nodepool.getHoldRequests() self.assertEqual(1, len(request_list)) request = self.sched_zk_nodepool.getHoldRequest( request_list[0]) self.assertIsNotNone(request) request = self.scheds.first.sched.autohold_info(request.id) self.assertNotEqual({}, request) self.assertEqual('tenant-one', request['tenant']) self.assertEqual('review.example.com/org/project', request['project']) self.assertEqual('project-test2', request['job']) self.assertEqual('reason text', request['reason']) self.assertEqual(1, request['max_count']) self.assertEqual(0, request['current_count']) @simple_layout('layouts/autohold.yaml') def test_autohold_delete(self): self.scheds.first.sched.autohold( 'tenant-one', 'review.example.com/org/project', 'project-test2', ".*", "reason text", 1, None) # There should be a record in ZooKeeper request_list = self.sched_zk_nodepool.getHoldRequests() self.assertEqual(1, len(request_list)) request = self.sched_zk_nodepool.getHoldRequest( request_list[0]) self.assertIsNotNone(request) # Delete and verify no more requests self.scheds.first.sched.autohold_delete(request.id) request_list = self.sched_zk_nodepool.getHoldRequests() self.assertEqual([], request_list) def test_autohold_padding(self): self.scheds.first.sched.autohold( 'tenant-one', 'review.example.com/org/project', 'project-test2', ".*", "reason text", 1, None) # There should be a record in ZooKeeper request_list = self.sched_zk_nodepool.getHoldRequests() self.assertEqual(1, len(request_list)) request = self.sched_zk_nodepool.getHoldRequest( request_list[0]) self.assertIsNotNone(request) # Assert the ID leads with a bunch of zeros, them strip them # off to test autohold_delete can handle a user passing in an # ID without leading zeros. self.assertEqual(request.id[0:5], '00000') trimmed_request = request.id[5:] # Delete and verify no more requests self.scheds.first.sched.autohold_delete(trimmed_request) request_list = self.sched_zk_nodepool.getHoldRequests() self.assertEqual([], request_list) def _test_autohold_scoped(self, change_obj, change, ref): # create two changes on the same project, and autohold request # for one of them. other = self.fake_gerrit.addFakeChange( 'org/project', 'master', 'other' ) if change != "": ref = "refs/changes/%s/%s/.*" % ( str(change_obj.number).zfill(2)[-2:], str(change_obj.number) ) self.scheds.first.sched.autohold( 'tenant-one', 'review.example.com/org/project', 'project-test2', ref, "reason text", 1, None) # First, check that an unrelated job does not trigger autohold, even # when it failed self.executor_server.failJob('project-test2', other) self.fake_gerrit.addEvent(other.getPatchsetCreatedEvent(1)) self.waitUntilSettled() self.assertEqual(other.data['status'], 'NEW') self.assertEqual(other.reported, 1) # project-test2 self.assertEqual(self.history[0].result, 'FAILURE') # Check nodepool for a held node held_node = None for node in self.fake_nodepool.getNodes(): if node['state'] == zuul.model.STATE_HOLD: held_node = node break self.assertIsNone(held_node) # And then verify that failed job for the defined change # triggers the autohold self.executor_server.failJob('project-test2', change_obj) self.fake_gerrit.addEvent(change_obj.getPatchsetCreatedEvent(1)) self.waitUntilSettled() self.assertEqual(change_obj.data['status'], 'NEW') self.assertEqual(change_obj.reported, 1) # project-test2 self.assertEqual(self.history[1].result, 'FAILURE') # Check nodepool for a held node held_node = None for node in self.fake_nodepool.getNodes(): if node['state'] == zuul.model.STATE_HOLD: held_node = node break self.assertIsNotNone(held_node) # Validate node has recorded the failed job self.assertEqual( held_node['hold_job'], " ".join(['tenant-one', 'review.example.com/org/project', 'project-test2', ref]) ) self.assertEqual(held_node['comment'], "reason text") @simple_layout('layouts/autohold.yaml') def test_autohold_change(self): A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') self._test_autohold_scoped(A, change=A.number, ref="") @simple_layout('layouts/autohold.yaml') def test_autohold_ref(self): A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') ref = A.data['currentPatchSet']['ref'] self._test_autohold_scoped(A, change="", ref=ref) @simple_layout('layouts/autohold.yaml') def test_autohold_scoping(self): A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') # create three autohold requests, scoped to job, change and # a specific ref change = str(A.number) change_ref = "refs/changes/%s/%s/.*" % ( str(change).zfill(2)[-2:], str(change) ) ref = A.data['currentPatchSet']['ref'] self.scheds.first.sched.autohold( 'tenant-one', 'review.example.com/org/project', 'project-test2', ".*", "reason text", 1, None) self.scheds.first.sched.autohold( 'tenant-one', 'review.example.com/org/project', 'project-test2', change_ref, "reason text", 1, None) self.scheds.first.sched.autohold( 'tenant-one', 'review.example.com/org/project', 'project-test2', ref, "reason text", 1, None) # Fail 3 jobs for the same change, and verify that the autohold # requests are fullfilled in the expected order: from the most # specific towards the most generic one. def _fail_job_and_verify_autohold_request(change_obj, ref_filter): self.executor_server.failJob('project-test2', change_obj) self.fake_gerrit.addEvent(change_obj.getPatchsetCreatedEvent(1)) self.waitUntilSettled() # Check nodepool for a held node held_node = None for node in self.fake_nodepool.getNodes(): if node['state'] == zuul.model.STATE_HOLD: held_node = node break self.assertIsNotNone(held_node) self.assertEqual( held_node['hold_job'], " ".join(['tenant-one', 'review.example.com/org/project', 'project-test2', ref_filter]) ) self.assertFalse(held_node['_lock'], "Node %s is locked" % (node['_oid'],)) self.fake_nodepool.removeNode(held_node) _fail_job_and_verify_autohold_request(A, ref) ref = "refs/changes/%s/%s/.*" % (str(change).zfill(2)[-2:], str(change)) _fail_job_and_verify_autohold_request(A, ref) _fail_job_and_verify_autohold_request(A, ".*") @simple_layout('layouts/autohold.yaml') def test_autohold_ignores_aborted_jobs(self): self.scheds.first.sched.autohold( 'tenant-one', 'review.example.com/org/project', 'project-test2', ".*", "reason text", 1, None) self.executor_server.hold_jobs_in_build = True # Create a change that will have its job aborted A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) self.waitUntilSettled() # Creating new patchset on change A will abort A,1's job because # a new patchset arrived replacing A,1 with A,2. A.addPatchset() self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(2)) self.waitUntilSettled() self.executor_server.hold_jobs_in_build = False self.executor_server.release() self.waitUntilSettled() self.assertEqual(A.data['status'], 'NEW') # Note only the successful job for A,2 will report as we don't # report aborted builds for old patchsets. self.assertEqual(A.reported, 1) # A,1 project-test2 self.assertEqual(self.history[0].result, 'ABORTED') # A,2 project-test2 self.assertEqual(self.history[1].result, 'SUCCESS') # Check nodepool for a held node held_node = None for node in self.fake_nodepool.getNodes(): if node['state'] == zuul.model.STATE_HOLD: held_node = node break self.assertIsNone(held_node) @simple_layout('layouts/autohold.yaml') def test_autohold_hold_expiration(self): self.scheds.first.sched.autohold( 'tenant-one', 'review.example.com/org/project', 'project-test2', ".*", "reason text", 1, 30) # Hold a failed job B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B') self.executor_server.failJob('project-test2', B) self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1)) self.waitUntilSettled() self.assertEqual(B.data['status'], 'NEW') self.assertEqual(B.reported, 1) # project-test2 self.assertEqual(self.history[0].result, 'FAILURE') # Check nodepool for a held node held_node = None for node in self.fake_nodepool.getNodes(): if node['state'] == zuul.model.STATE_HOLD: held_node = node break self.assertIsNotNone(held_node) # Validate node has hold_expiration property self.assertEqual(int(held_node['hold_expiration']), 30) @simple_layout('layouts/autohold.yaml') def test_autohold_list(self): self.scheds.first.sched.autohold( 'tenant-one', 'review.example.com/org/project', 'project-test2', ".*", "reason text", 1, None) autohold_requests = self.scheds.first.sched.autohold_list() self.assertNotEqual([], autohold_requests) self.assertEqual(1, len(autohold_requests)) request = autohold_requests[0] self.assertEqual('tenant-one', request['tenant']) self.assertIn('org/project', request['project']) self.assertEqual('project-test2', request['job']) self.assertEqual(".*", request['ref_filter']) self.assertEqual("reason text", request['reason']) @simple_layout('layouts/autohold.yaml') def test_autohold_request_expiration(self): orig_exp = RecordingExecutorServer.EXPIRED_HOLD_REQUEST_TTL def reset_exp(): self.executor_server.EXPIRED_HOLD_REQUEST_TTL = orig_exp self.addCleanup(reset_exp) # Temporarily shorten the hold request expiration time self.scheds.first.sched.autohold( 'tenant-one', 'review.example.com/org/project', 'project-test2', ".*", "reason text", 1, 1) autohold_requests = self.scheds.first.sched.autohold_list() self.assertEqual(1, len(autohold_requests)) req = autohold_requests[0] self.assertIsNone(req['expired']) A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') self.executor_server.failJob('project-test2', A) self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) self.waitUntilSettled() autohold_requests = self.scheds.first.sched.autohold_list() self.assertEqual(1, len(autohold_requests)) req = autohold_requests[0] self.assertIsNotNone(req['expired']) # Temporarily shorten hold time so that the hold request can be # auto-deleted (which is done on another test failure). And wait # long enough for nodes to expire and request to delete. self.executor_server.EXPIRED_HOLD_REQUEST_TTL = 1 time.sleep(3) B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B') self.executor_server.failJob('project-test2', B) self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1)) self.waitUntilSettled() for _ in iterate_timeout(10, 'hold request expiration'): if len(self.scheds.first.sched.autohold_list()) == 0: break @simple_layout('layouts/three-projects.yaml') def test_dependent_behind_dequeue(self): # This particular test does a large amount of merges and needs a little # more time to complete self.wait_timeout = 120 "test that dependent changes behind dequeued changes work" # This complicated test is a reproduction of a real life bug self.scheds.execute(lambda app: app.sched.reconfigure(app.config)) self.executor_server.hold_jobs_in_build = True A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A') B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B') C = self.fake_gerrit.addFakeChange('org/project2', 'master', 'C') D = self.fake_gerrit.addFakeChange('org/project2', 'master', 'D') E = self.fake_gerrit.addFakeChange('org/project2', 'master', 'E') F = self.fake_gerrit.addFakeChange('org/project3', 'master', 'F') D.setDependsOn(C, 1) E.setDependsOn(D, 1) A.addApproval('Code-Review', 2) B.addApproval('Code-Review', 2) C.addApproval('Code-Review', 2) D.addApproval('Code-Review', 2) E.addApproval('Code-Review', 2) F.addApproval('Code-Review', 2) A.fail_merge = True # Change object re-use in the gerrit trigger is hidden if # changes are added in quick succession; waiting makes it more # like real life. self.fake_gerrit.addEvent(A.addApproval('Approved', 1)) self.waitUntilSettled() self.fake_gerrit.addEvent(B.addApproval('Approved', 1)) self.waitUntilSettled() self.executor_server.release('.*-merge') self.waitUntilSettled() self.executor_server.release('.*-merge') self.waitUntilSettled() self.fake_gerrit.addEvent(C.addApproval('Approved', 1)) self.waitUntilSettled() self.fake_gerrit.addEvent(D.addApproval('Approved', 1)) self.waitUntilSettled() self.fake_gerrit.addEvent(E.addApproval('Approved', 1)) self.waitUntilSettled() self.fake_gerrit.addEvent(F.addApproval('Approved', 1)) self.waitUntilSettled() self.executor_server.release('.*-merge') self.waitUntilSettled() self.executor_server.release('.*-merge') self.waitUntilSettled() self.executor_server.release('.*-merge') self.waitUntilSettled() self.executor_server.release('.*-merge') self.waitUntilSettled() # all jobs running # Grab pointers to the jobs we want to release before # releasing any, because list indexes may change as # the jobs complete. a, b, c = self.builds[:3] a.release() b.release() c.release() self.waitUntilSettled() self.executor_server.hold_jobs_in_build = False self.executor_server.release() self.waitUntilSettled() self.assertEqual(A.data['status'], 'NEW') self.assertEqual(B.data['status'], 'MERGED') self.assertEqual(C.data['status'], 'MERGED') self.assertEqual(D.data['status'], 'MERGED') self.assertEqual(E.data['status'], 'MERGED') self.assertEqual(F.data['status'], 'MERGED') self.assertEqual(A.reported, 2) self.assertEqual(B.reported, 2) self.assertEqual(C.reported, 2) self.assertEqual(D.reported, 2) self.assertEqual(E.reported, 2) self.assertEqual(F.reported, 2) self.assertEqual(self.countJobResults(self.history, 'ABORTED'), 15) self.assertEqual(len(self.history), 44) def test_merger_repack(self): "Test that the merger works after a repack" A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') A.addApproval('Code-Review', 2) self.fake_gerrit.addEvent(A.addApproval('Approved', 1)) self.waitUntilSettled() self.assertEqual(self.getJobFromHistory('project-merge').result, 'SUCCESS') self.assertEqual(self.getJobFromHistory('project-test1').result, 'SUCCESS') self.assertEqual(self.getJobFromHistory('project-test2').result, 'SUCCESS') self.assertEqual(A.data['status'], 'MERGED') self.assertEqual(A.reported, 2) self.assertEmptyQueues() self.build_history = [] path = os.path.join(self.merger_src_root, "review.example.com", "org/project") if os.path.exists(path): repack_repo(path) path = os.path.join(self.executor_src_root, "review.example.com", "org/project") if os.path.exists(path): repack_repo(path) A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') A.addApproval('Code-Review', 2) self.fake_gerrit.addEvent(A.addApproval('Approved', 1)) self.waitUntilSettled() self.assertEqual(self.getJobFromHistory('project-merge').result, 'SUCCESS') self.assertEqual(self.getJobFromHistory('project-test1').result, 'SUCCESS') self.assertEqual(self.getJobFromHistory('project-test2').result, 'SUCCESS') self.assertEqual(A.data['status'], 'MERGED') self.assertEqual(A.reported, 2) def test_merger_repack_large_change(self): "Test that the merger works with large changes after a repack" # https://bugs.executepad.net/zuul/+bug/1078946 # This test assumes the repo is already cloned; make sure it is tenant = self.scheds.first.sched.abide.tenants.get('tenant-one') trusted, project = tenant.getProject('org/project') url = self.fake_gerrit.getGitUrl(project) self.executor_server.merger._addProject( 'review.example.com', 'gerrit', 'org/project', url, None, None, None) A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') A.addPatchset(large=True) # TODOv3(jeblair): add hostname to upstream root path = os.path.join(self.upstream_root, 'org/project') repack_repo(path) path = os.path.join(self.merger_src_root, 'review.example.com', 'org/project') if os.path.exists(path): repack_repo(path) path = os.path.join(self.executor_src_root, 'review.example.com', 'org/project') if os.path.exists(path): repack_repo(path) A.addApproval('Code-Review', 2) self.fake_gerrit.addEvent(A.addApproval('Approved', 1)) self.waitUntilSettled() self.assertEqual(self.getJobFromHistory('project-merge').result, 'SUCCESS') self.assertEqual(self.getJobFromHistory('project-test1').result, 'SUCCESS') self.assertEqual(self.getJobFromHistory('project-test2').result, 'SUCCESS') self.assertEqual(A.data['status'], 'MERGED') self.assertEqual(A.reported, 2) def test_new_patchset_dequeues_old(self): "Test that a new patchset causes the old to be dequeued" # D -> C (depends on B) -> B (depends on A) -> A -> M self.executor_server.hold_jobs_in_build = True M = self.fake_gerrit.addFakeChange('org/project', 'master', 'M') M.setMerged() A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B') C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C') D = self.fake_gerrit.addFakeChange('org/project', 'master', 'D') A.addApproval('Code-Review', 2) B.addApproval('Code-Review', 2) C.addApproval('Code-Review', 2) D.addApproval('Code-Review', 2) C.setDependsOn(B, 1) B.setDependsOn(A, 1) A.setDependsOn(M, 1) self.fake_gerrit.addEvent(A.addApproval('Approved', 1)) self.fake_gerrit.addEvent(B.addApproval('Approved', 1)) self.fake_gerrit.addEvent(C.addApproval('Approved', 1)) self.fake_gerrit.addEvent(D.addApproval('Approved', 1)) self.waitUntilSettled() B.addPatchset() self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(2)) self.waitUntilSettled() self.executor_server.hold_jobs_in_build = False self.executor_server.release() self.waitUntilSettled() self.assertEqual(A.data['status'], 'MERGED') self.assertEqual(A.reported, 2) self.assertEqual(B.data['status'], 'NEW') self.assertEqual(B.reported, 2) self.assertEqual(C.data['status'], 'NEW') self.assertEqual(C.reported, 2) self.assertEqual(D.data['status'], 'MERGED') self.assertEqual(D.reported, 2) self.assertEqual(len(self.history), 9) # 3 each for A, B, D. @simple_layout('layouts/no-dequeue-on-new-patchset.yaml') def test_no_dequeue_on_new_patchset(self): "Test the dequeue-on-new-patchset false value" A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A') self.executor_server.hold_jobs_in_build = True A.addApproval('Code-Review', 2) self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) self.waitUntilSettled() self.assertEqual(len(self.builds), 1) A.addPatchset() A.addApproval('Code-Review', 2) self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(2)) self.waitUntilSettled() self.assertEqual(len(self.builds), 2) self.executor_server.hold_jobs_in_build = False self.executor_server.release() self.waitUntilSettled() self.assertHistory([ dict(name='project1-test', result='SUCCESS', changes='1,1'), dict(name='project1-test', result='SUCCESS', changes='1,2'), ], ordered=False) @simple_layout('layouts/no-dequeue-on-new-patchset.yaml') def test_no_dequeue_on_new_patchset_deps(self): "Test dependencies are updated if dequeue-on-new-patchset is false" A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A') B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B') B.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % ( B.subject, A.data['url']) A.addApproval('Code-Review', 2) B.addApproval('Code-Review', 2) self.executor_server.hold_jobs_in_build = True self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1)) self.waitUntilSettled() self.assertEqual(len(self.builds), 1) A.addPatchset() self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(2)) self.waitUntilSettled() self.assertEqual(len(self.builds), 0) self.executor_server.hold_jobs_in_build = False self.executor_server.release() self.waitUntilSettled() # The item should be dequeued because of the dependency change self.assertHistory([ dict(name='project1-test', result='ABORTED', changes='1,1 2,1'), ], ordered=False) def test_new_patchset_check(self): "Test a new patchset in check" self.executor_server.hold_jobs_in_build = True A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B') tenant = self.scheds.first.sched.abide.tenants.get('tenant-one') check_pipeline = tenant.layout.pipelines['check'] # Add two git-dependent changes B.setDependsOn(A, 1) self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1)) self.waitUntilSettled() self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) self.waitUntilSettled() # A live item, and a non-live/live pair items = check_pipeline.getAllItems() self.assertEqual(len(items), 3) self.assertEqual(items[0].changes[0].number, '1') self.assertEqual(items[0].changes[0].patchset, '1') self.assertFalse(items[0].live) self.assertEqual(items[1].changes[0].number, '2') self.assertEqual(items[1].changes[0].patchset, '1') self.assertTrue(items[1].live) self.assertEqual(items[2].changes[0].number, '1') self.assertEqual(items[2].changes[0].patchset, '1') self.assertTrue(items[2].live) # Add a new patchset to A A.addPatchset() self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(2)) self.waitUntilSettled() # The live copy of A,1 should be gone, but the non-live and B # should continue, and we should have a new A,2 items = check_pipeline.getAllItems() self.assertEqual(len(items), 3) self.assertEqual(items[0].changes[0].number, '1') self.assertEqual(items[0].changes[0].patchset, '1') self.assertFalse(items[0].live) self.assertEqual(items[1].changes[0].number, '2') self.assertEqual(items[1].changes[0].patchset, '1') self.assertTrue(items[1].live) self.assertEqual(items[2].changes[0].number, '1') self.assertEqual(items[2].changes[0].patchset, '2') self.assertTrue(items[2].live) # Add a new patchset to B B.addPatchset() self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(2)) self.waitUntilSettled() # The live copy of B,1 should be gone, and it's non-live copy of A,1 # but we should have a new B,2 (still based on A,1) items = check_pipeline.getAllItems() self.assertEqual(len(items), 3) self.assertEqual(items[0].changes[0].number, '1') self.assertEqual(items[0].changes[0].patchset, '2') self.assertTrue(items[0].live) self.assertEqual(items[1].changes[0].number, '1') self.assertEqual(items[1].changes[0].patchset, '1') self.assertFalse(items[1].live) self.assertEqual(items[2].changes[0].number, '2') self.assertEqual(items[2].changes[0].patchset, '2') self.assertTrue(items[2].live) self.builds[0].release() self.waitUntilSettled() self.builds[0].release() self.waitUntilSettled() self.executor_server.hold_jobs_in_build = False self.executor_server.release() self.waitUntilSettled() self.assertEqual(A.reported, 1) self.assertEqual(B.reported, 1) self.assertEqual(self.history[0].result, 'ABORTED') self.assertEqual(self.history[0].changes, '1,1') self.assertEqual(self.history[1].result, 'ABORTED') self.assertEqual(self.history[1].changes, '1,1 2,1') self.assertEqual(self.history[2].result, 'SUCCESS') self.assertEqual(self.history[2].changes, '1,2') self.assertEqual(self.history[3].result, 'SUCCESS') self.assertEqual(self.history[3].changes, '1,1 2,2') def test_abandoned_gate(self): "Test that an abandoned change is dequeued from gate" self.executor_server.hold_jobs_in_build = True A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') A.addApproval('Code-Review', 2) self.fake_gerrit.addEvent(A.addApproval('Approved', 1)) self.waitUntilSettled() self.assertEqual(len(self.builds), 1, "One job being built (on hold)") self.assertEqual(self.builds[0].name, 'project-merge') self.fake_gerrit.addEvent(A.getChangeAbandonedEvent()) self.waitUntilSettled() self.executor_server.release('.*-merge') self.waitUntilSettled() self.assertBuilds([]) self.assertHistory([ dict(name='project-merge', result='ABORTED', changes='1,1')], ordered=False) self.assertEqual(A.reported, 1, "Abandoned gate change should report only start") def test_abandoned_check(self): "Test that an abandoned change is dequeued from check" self.executor_server.hold_jobs_in_build = True A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B') tenant = self.scheds.first.sched.abide.tenants.get('tenant-one') check_pipeline = tenant.layout.pipelines['check'] # Add two git-dependent changes B.setDependsOn(A, 1) self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1)) self.waitUntilSettled() self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) self.waitUntilSettled() # A live item, and a non-live/live pair items = check_pipeline.getAllItems() self.assertEqual(len(items), 3) self.assertEqual(items[0].changes[0].number, '1') self.assertFalse(items[0].live) self.assertEqual(items[1].changes[0].number, '2') self.assertTrue(items[1].live) self.assertEqual(items[2].changes[0].number, '1') self.assertTrue(items[2].live) # Abandon A self.fake_gerrit.addEvent(A.getChangeAbandonedEvent()) self.waitUntilSettled() # The live copy of A should be gone, but the non-live and B # should continue items = check_pipeline.getAllItems() self.assertEqual(len(items), 2) self.assertEqual(items[0].changes[0].number, '1') self.assertFalse(items[0].live) self.assertEqual(items[1].changes[0].number, '2') self.assertTrue(items[1].live) self.executor_server.hold_jobs_in_build = False self.executor_server.release() self.waitUntilSettled() self.assertEqual(len(self.history), 4) self.assertEqual(self.history[0].result, 'ABORTED', 'Build should have been aborted') self.assertEqual(A.reported, 0, "Abandoned change should not report") self.assertEqual(B.reported, 1, "Change should report") def test_cancel_starting_build(self): "Test that a canceled build that is not processed yet is removed" self.executor_server.hold_jobs_in_start = True A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) for _ in iterate_timeout(30, 'Wait for build to be in starting phase'): if self.executor_server.job_workers: break tevent = threading.Event() def data_watch(event): # Set the threading event as soon as the cancel node is present tevent.set() builds = list(self.executor_api.all()) # When using DataWatch it is possible for the cancel znode to be # created, then deleted almost immediately before the watch handling # happens. When this happens kazoo sees no stat info and treats the # event as a noop because no new version is available. Use exists to # avoid this problem. self.zk_client.client.exists(f"{builds[0].path}/cancel", data_watch) # Abandon change to cancel build self.fake_gerrit.addEvent(A.getChangeAbandonedEvent()) self.assertTrue(tevent.wait(timeout=30)) self.executor_server.hold_jobs_in_start = False self.waitUntilSettled() self.assertHistory([ dict(name='project-merge', result='ABORTED') ]) def test_abandoned_not_timer(self): "Test that an abandoned change does not cancel timer jobs" # This test can not use simple_layout because it must start # with a configuration which does not include a # timer-triggered job so that we have an opportunity to set # the hold flag before the first job. self.executor_server.hold_jobs_in_build = True # Start timer trigger - also org/project self.commitConfigUpdate('common-config', 'layouts/idle.yaml') self.scheds.execute(lambda app: app.sched.reconfigure(app.config)) # The pipeline triggers every second, so we should have seen # several by now. time.sleep(5) self.waitUntilSettled() # Stop queuing timer triggered jobs so that the assertions # below don't race against more jobs being queued. # Must be in same repo, so overwrite config with another one self.commitConfigUpdate('common-config', 'layouts/no-timer.yaml') self.scheds.execute(lambda app: app.sched.reconfigure(app.config)) self.waitUntilSettled() # If APScheduler is in mid-event when we remove the job, we # can end up with one more event firing, so give it an extra # second to settle. time.sleep(1) self.waitUntilSettled() self.assertEqual(len(self.builds), 1, "One timer job") A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) self.waitUntilSettled() self.assertEqual(len(self.builds), 2, "One change plus one timer job") self.fake_gerrit.addEvent(A.getChangeAbandonedEvent()) self.waitUntilSettled() self.assertEqual(len(self.builds), 1, "One timer job remains") self.executor_server.release() self.waitUntilSettled() def test_new_patchset_dequeues_old_on_head(self): "Test that a new patchset causes the old to be dequeued (at head)" # D -> C (depends on B) -> B (depends on A) -> A -> M self.executor_server.hold_jobs_in_build = True M = self.fake_gerrit.addFakeChange('org/project', 'master', 'M') M.setMerged() A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B') C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C') D = self.fake_gerrit.addFakeChange('org/project', 'master', 'D') A.addApproval('Code-Review', 2) B.addApproval('Code-Review', 2) C.addApproval('Code-Review', 2) D.addApproval('Code-Review', 2) C.setDependsOn(B, 1) B.setDependsOn(A, 1) A.setDependsOn(M, 1) self.fake_gerrit.addEvent(A.addApproval('Approved', 1)) self.fake_gerrit.addEvent(B.addApproval('Approved', 1)) self.fake_gerrit.addEvent(C.addApproval('Approved', 1)) self.fake_gerrit.addEvent(D.addApproval('Approved', 1)) self.waitUntilSettled() A.addPatchset() self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(2)) self.waitUntilSettled() self.executor_server.hold_jobs_in_build = False self.executor_server.release() self.waitUntilSettled() self.assertEqual(A.data['status'], 'NEW') self.assertEqual(A.reported, 2) self.assertEqual(B.data['status'], 'NEW') self.assertEqual(B.reported, 2) self.assertEqual(C.data['status'], 'NEW') self.assertEqual(C.reported, 2) self.assertEqual(D.data['status'], 'MERGED') self.assertEqual(D.reported, 2) self.assertEqual(len(self.history), 7) def test_new_patchset_dequeues_old_without_dependents(self): "Test that a new patchset causes only the old to be dequeued" self.executor_server.hold_jobs_in_build = True A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B') C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C') A.addApproval('Code-Review', 2) B.addApproval('Code-Review', 2) C.addApproval('Code-Review', 2) self.fake_gerrit.addEvent(C.addApproval('Approved', 1)) self.fake_gerrit.addEvent(B.addApproval('Approved', 1)) self.fake_gerrit.addEvent(A.addApproval('Approved', 1)) self.waitUntilSettled() B.addPatchset() self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(2)) self.waitUntilSettled() self.executor_server.hold_jobs_in_build = False self.executor_server.release() self.waitUntilSettled() self.assertEqual(A.data['status'], 'MERGED') self.assertEqual(A.reported, 2) self.assertEqual(B.data['status'], 'NEW') self.assertEqual(B.reported, 2) self.assertEqual(C.data['status'], 'MERGED') self.assertEqual(C.reported, 2) self.assertEqual(len(self.history), 9) def test_new_patchset_dequeues_old_independent_queue(self): "Test that a new patchset causes the old to be dequeued (independent)" self.executor_server.hold_jobs_in_build = True A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B') C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C') self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1)) self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1)) self.waitUntilSettled() B.addPatchset() self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(2)) self.waitUntilSettled() self.executor_server.hold_jobs_in_build = False self.executor_server.release() self.waitUntilSettled() self.assertEqual(A.data['status'], 'NEW') self.assertEqual(A.reported, 1) self.assertEqual(B.data['status'], 'NEW') self.assertEqual(B.reported, 1) self.assertEqual(C.data['status'], 'NEW') self.assertEqual(C.reported, 1) self.assertEqual(len(self.history), 10) self.assertEqual(self.countJobResults(self.history, 'ABORTED'), 1) @simple_layout('layouts/noop-job.yaml') def test_noop_job(self): "Test that the internal noop job works" A = self.fake_gerrit.addFakeChange('org/noop-project', 'master', 'A') A.addApproval('Code-Review', 2) self.fake_gerrit.addEvent(A.addApproval('Approved', 1)) self.waitUntilSettled() queue = list(self.executor_api.queued()) self.assertEqual(len(queue), 0) self.assertTrue(self.scheds.first.sched._areAllBuildsComplete()) self.assertEqual(len(self.history), 0) self.assertEqual(A.data['status'], 'MERGED') self.assertEqual(A.reported, 2) @simple_layout('layouts/no-jobs-project.yaml') def test_no_job_project(self): "Test that reports with no jobs don't get sent" A = self.fake_gerrit.addFakeChange('org/no-jobs-project', 'master', 'A') self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) self.waitUntilSettled() # Change wasn't reported to self.assertEqual(A.reported, False) # Check queue is empty afterwards tenant = self.scheds.first.sched.abide.tenants.get('tenant-one') check_pipeline = tenant.layout.pipelines['check'] items = check_pipeline.getAllItems() self.assertEqual(len(items), 0) self.assertEqual(len(self.history), 0) def test_zuul_refs(self): "Test that zuul refs exist and have the right changes" self.executor_server.hold_jobs_in_build = True M1 = self.fake_gerrit.addFakeChange('org/project1', 'master', 'M1') M1.setMerged() M2 = self.fake_gerrit.addFakeChange('org/project2', 'master', 'M2') M2.setMerged() A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A') B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B') C = self.fake_gerrit.addFakeChange('org/project2', 'master', 'C') D = self.fake_gerrit.addFakeChange('org/project2', 'master', 'D') A.addApproval('Code-Review', 2) B.addApproval('Code-Review', 2) C.addApproval('Code-Review', 2) D.addApproval('Code-Review', 2) self.fake_gerrit.addEvent(A.addApproval('Approved', 1)) self.fake_gerrit.addEvent(B.addApproval('Approved', 1)) self.fake_gerrit.addEvent(C.addApproval('Approved', 1)) self.fake_gerrit.addEvent(D.addApproval('Approved', 1)) self.waitUntilSettled() self.executor_server.release('.*-merge') self.waitUntilSettled() self.executor_server.release('.*-merge') self.waitUntilSettled() self.executor_server.release('.*-merge') self.waitUntilSettled() self.executor_server.release('.*-merge') self.waitUntilSettled() a_build = b_build = c_build = d_build = None for x in self.builds: if x.parameters['zuul']['change'] == '3': a_build = x elif x.parameters['zuul']['change'] == '4': b_build = x elif x.parameters['zuul']['change'] == '5': c_build = x elif x.parameters['zuul']['change'] == '6': d_build = x if a_build and b_build and c_build and d_build: break # should have a, not b, and should not be in project2 self.assertTrue(a_build.hasChanges(A)) self.assertFalse(a_build.hasChanges(B, M2)) # should have a and b, and should not be in project2 self.assertTrue(b_build.hasChanges(A, B)) self.assertFalse(b_build.hasChanges(M2)) # should have a and b in 1, c in 2 self.assertTrue(c_build.hasChanges(A, B, C)) self.assertFalse(c_build.hasChanges(D)) # should have a and b in 1, c and d in 2 self.assertTrue(d_build.hasChanges(A, B, C, D)) self.executor_server.hold_jobs_in_build = False self.executor_server.release() self.waitUntilSettled() self.assertEqual(A.data['status'], 'MERGED') self.assertEqual(A.reported, 2) self.assertEqual(B.data['status'], 'MERGED') self.assertEqual(B.reported, 2) self.assertEqual(C.data['status'], 'MERGED') self.assertEqual(C.reported, 2) self.assertEqual(D.data['status'], 'MERGED') self.assertEqual(D.reported, 2) def test_rerun_on_error(self): "Test that if a worker fails to run a job, it is run again" self.executor_server.hold_jobs_in_build = True A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') A.addApproval('Code-Review', 2) self.fake_gerrit.addEvent(A.addApproval('Approved', 1)) self.waitUntilSettled() self.builds[0].requeue = True self.executor_server.hold_jobs_in_build = False self.executor_server.release() self.waitUntilSettled() self.assertEqual(self.countJobResults(self.history, None), 1) self.assertEqual(self.countJobResults(self.history, 'SUCCESS'), 3) def test_statsd(self): "Test each of the statsd methods used in the scheduler" statsd = self.scheds.first.sched.statsd statsd.incr('test-incr') statsd.timing('test-timing', 3) statsd.gauge('test-gauge', 12) self.assertReportedStat('test-incr', '1', 'c') self.assertReportedStat('test-timing', '3', 'ms') self.assertReportedStat('test-gauge', '12', 'g') # test key normalization statsd.extra_keys = {'hostname': '1_2_3_4'} statsd.incr('hostname-incr.{hostname}.{fake}', fake='1:2') statsd.timing('hostname-timing.{hostname}.{fake}', 3, fake='1:2') statsd.gauge('hostname-gauge.{hostname}.{fake}', 12, fake='1:2') self.assertReportedStat('hostname-incr.1_2_3_4.1_2', '1', 'c') self.assertReportedStat('hostname-timing.1_2_3_4.1_2', '3', 'ms') self.assertReportedStat('hostname-gauge.1_2_3_4.1_2', '12', 'g') def test_statsd_conflict(self): statsd = self.scheds.first.sched.statsd statsd.gauge('test-gauge', 12) # since test-gauge is already a value, we can't make # subvalues. Test the assert works. statsd.gauge('test-gauge.1_2_3_4', 12) self.assertReportedStat('test-gauge', '12', 'g') self.assertRaises(Exception, self.assertReportedStat, 'test-gauge.1_2_3_4', '12', 'g') def test_file_head(self): # This is a regression test for an observed bug. A change # with a file named "HEAD" in the root directory of the repo # was processed by a merger. It then was unable to reset the # repo because of: # GitCommandError: 'git reset --hard HEAD' returned # with exit code 128 # stderr: 'fatal: ambiguous argument 'HEAD': both revision # and filename # Use '--' to separate filenames from revisions' A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') A.addPatchset({'HEAD': ''}) B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B') self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(2)) self.waitUntilSettled() self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1)) self.waitUntilSettled() self.assertIn('Build succeeded', A.messages[0]) self.assertIn('Build succeeded', B.messages[0]) def test_file_jobs(self): "Test that file jobs run only when appropriate" A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') A.addPatchset({'pip-requires': 'foo'}) B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B') A.addApproval('Code-Review', 2) B.addApproval('Code-Review', 2) self.fake_gerrit.addEvent(A.addApproval('Approved', 1)) self.fake_gerrit.addEvent(B.addApproval('Approved', 1)) self.waitUntilSettled() testfile_jobs = [x for x in self.history if x.name == 'project-testfile'] self.assertEqual(len(testfile_jobs), 1) self.assertEqual(testfile_jobs[0].changes, '1,2') self.assertEqual(A.data['status'], 'MERGED') self.assertEqual(A.reported, 2) self.assertEqual(B.data['status'], 'MERGED') self.assertEqual(B.reported, 2) def _test_irrelevant_files_jobs(self, should_skip): "Test that jobs with irrelevant-files filter run only when appropriate" if should_skip: files = {'ignoreme': 'ignored\n'} else: files = {'respectme': 'please!\n'} change = self.fake_gerrit.addFakeChange('org/project', 'master', 'test irrelevant-files', files=files) self.fake_gerrit.addEvent(change.getPatchsetCreatedEvent(1)) self.waitUntilSettled() tested_change_ids = [x.changes[0] for x in self.history if x.name == 'project-test-irrelevant-files'] if should_skip: self.assertEqual([], tested_change_ids) else: self.assertIn(change.data['number'], tested_change_ids) @simple_layout('layouts/irrelevant-files.yaml') def test_irrelevant_files_match_skips_job(self): self._test_irrelevant_files_jobs(should_skip=True) @simple_layout('layouts/irrelevant-files.yaml') def test_irrelevant_files_no_match_runs_job(self): self._test_irrelevant_files_jobs(should_skip=False) @simple_layout('layouts/inheritance.yaml') def test_inherited_jobs_keep_matchers(self): files = {'ignoreme': 'ignored\n'} change = self.fake_gerrit.addFakeChange('org/project', 'master', 'test irrelevant-files', files=files) self.fake_gerrit.addEvent(change.getPatchsetCreatedEvent(1)) self.waitUntilSettled() run_jobs = set([build.name for build in self.history]) self.assertEqual(set(['project-test-nomatch-starts-empty', 'project-test-nomatch-starts-full']), run_jobs) @simple_layout('layouts/job-vars.yaml') def test_inherited_job_variables(self): A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) self.waitUntilSettled() self.assertHistory([ dict(name='parentjob', result='SUCCESS'), dict(name='child1', result='SUCCESS'), dict(name='child2', result='SUCCESS'), dict(name='child3', result='SUCCESS'), dict(name='override_project_var', result='SUCCESS'), dict(name='job_from_template1', result='SUCCESS'), dict(name='job_from_template2', result='SUCCESS'), ], ordered=False) j = self.getJobFromHistory('parentjob') rp = set([p['name'] for p in j.parameters['projects']]) job_vars = j.job.variables self.assertEqual(job_vars['project_var'], 'set_in_project') self.assertEqual(job_vars['template_var1'], 'set_in_template1') self.assertEqual(job_vars['template_var2'], 'set_in_template2') self.assertEqual(job_vars['override'], 0) self.assertEqual(job_vars['child1override'], 0) self.assertEqual(job_vars['parent'], 0) self.assertEqual(job_vars['deep']['override'], 0) self.assertFalse('child1' in job_vars) self.assertFalse('child2' in job_vars) self.assertFalse('child3' in job_vars) self.assertEqual(rp, set(['org/project', 'org/project0', 'org/project0'])) j = self.getJobFromHistory('child1') rp = set([p['name'] for p in j.parameters['projects']]) job_vars = j.job.variables self.assertEqual(job_vars['project_var'], 'set_in_project') self.assertEqual(job_vars['override'], 1) self.assertEqual(job_vars['child1override'], 1) self.assertEqual(job_vars['parent'], 0) self.assertEqual(job_vars['child1'], 1) self.assertEqual(job_vars['deep']['override'], 1) self.assertFalse('child2' in job_vars) self.assertFalse('child3' in job_vars) self.assertEqual(rp, set(['org/project', 'org/project0', 'org/project1'])) j = self.getJobFromHistory('child2') job_vars = j.job.variables self.assertEqual(job_vars['project_var'], 'set_in_project') rp = set([p['name'] for p in j.parameters['projects']]) self.assertEqual(job_vars['override'], 2) self.assertEqual(job_vars['child1override'], 0) self.assertEqual(job_vars['parent'], 0) self.assertEqual(job_vars['deep']['override'], 2) self.assertFalse('child1' in job_vars) self.assertEqual(job_vars['child2'], 2) self.assertFalse('child3' in job_vars) self.assertEqual(rp, set(['org/project', 'org/project0', 'org/project2'])) j = self.getJobFromHistory('child3') job_vars = j.job.variables self.assertEqual(job_vars['project_var'], 'set_in_project') rp = set([p['name'] for p in j.parameters['projects']]) self.assertEqual(job_vars['override'], 3) self.assertEqual(job_vars['child1override'], 0) self.assertEqual(job_vars['parent'], 0) self.assertEqual(job_vars['deep']['override'], 3) self.assertFalse('child1' in job_vars) self.assertFalse('child2' in job_vars) self.assertEqual(job_vars['child3'], 3) self.assertEqual(rp, set(['org/project', 'org/project0', 'org/project3'])) j = self.getJobFromHistory('override_project_var') job_vars = j.job.variables self.assertEqual(job_vars['project_var'], 'override_in_job') @simple_layout('layouts/job-variants.yaml') def test_job_branch_variants(self): self.create_branch('org/project', 'stable/diablo') self.fake_gerrit.addEvent( self.fake_gerrit.getFakeBranchCreatedEvent( 'org/project', 'stable/diablo')) self.create_branch('org/project', 'stable/essex') self.fake_gerrit.addEvent( self.fake_gerrit.getFakeBranchCreatedEvent( 'org/project', 'stable/essex')) self.waitUntilSettled() A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) self.waitUntilSettled() B = self.fake_gerrit.addFakeChange('org/project', 'stable/diablo', 'B') self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1)) self.waitUntilSettled() C = self.fake_gerrit.addFakeChange('org/project', 'stable/essex', 'C') self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1)) self.waitUntilSettled() self.assertHistory([ dict(name='python27', result='SUCCESS'), dict(name='python27', result='SUCCESS'), dict(name='python27', result='SUCCESS'), ]) j = self.history[0].job self.assertEqual(j.timeout, 40) self.assertEqual(len(j.nodeset.nodes), 1) self.assertEqual(next(iter(j.nodeset.nodes.values())).label, 'new') self.assertEqual([x['path'] for x in j.pre_run], ['base-pre', 'py27-pre']) self.assertEqual([x['path'] for x in j.post_run], ['py27-post-a', 'py27-post-b', 'base-post']) self.assertEqual([x['path'] for x in j.run], ['playbooks/python27.yaml']) j = self.history[1].job self.assertEqual(j.timeout, 50) self.assertEqual(len(j.nodeset.nodes), 1) self.assertEqual(next(iter(j.nodeset.nodes.values())).label, 'old') self.assertEqual([x['path'] for x in j.pre_run], ['base-pre', 'py27-pre', 'py27-diablo-pre']) self.assertEqual([x['path'] for x in j.post_run], ['py27-diablo-post', 'py27-post-a', 'py27-post-b', 'base-post']) self.assertEqual([x['path'] for x in j.run], ['py27-diablo']) j = self.history[2].job self.assertEqual(j.timeout, 40) self.assertEqual(len(j.nodeset.nodes), 1) self.assertEqual(next(iter(j.nodeset.nodes.values())).label, 'new') self.assertEqual([x['path'] for x in j.pre_run], ['base-pre', 'py27-pre', 'py27-essex-pre']) self.assertEqual([x['path'] for x in j.post_run], ['py27-essex-post', 'py27-post-a', 'py27-post-b', 'base-post']) self.assertEqual([x['path'] for x in j.run], ['playbooks/python27.yaml']) @simple_layout("layouts/no-run.yaml") def test_job_without_run(self): "Test that a job without a run playbook errors" A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) self.waitUntilSettled() self.assertIn('Job base does not specify a run playbook', A.messages[-1]) def test_queue_names(self): "Test shared change queue names" tenant = self.scheds.first.sched.abide.tenants.get('tenant-one') (trusted, project1) = tenant.getProject('org/project1') (trusted, project2) = tenant.getProject('org/project2') # Change queues are created lazy by the dependent pipeline manager # so retrieve the queue first without having to really enqueue a # change first. gate = tenant.layout.pipelines['gate'] FakeChange = namedtuple('FakeChange', ['project', 'branch']) fake_a = FakeChange(project1, 'master') fake_b = FakeChange(project2, 'master') with (pipeline_lock(self.zk_client, tenant.name, gate.name) as lock, self.createZKContext(lock) as ctx, gate.manager.currentContext(ctx)): gate.manager.getChangeQueue(fake_a, None) gate.manager.getChangeQueue(fake_b, None) q1 = gate.getQueue(project1.canonical_name, None) q2 = gate.getQueue(project2.canonical_name, None) self.assertEqual(q1.name, 'integrated') self.assertEqual(q2.name, 'integrated') @simple_layout("layouts/template-queue.yaml") def test_template_queue(self): "Test a shared queue can be constructed from a project-template" tenant = self.scheds.first.sched.abide.tenants.get('tenant-one') (trusted, project1) = tenant.getProject('org/project1') (trusted, project2) = tenant.getProject('org/project2') # Change queues are created lazy by the dependent pipeline manager # so retrieve the queue first without having to really enqueue a # change first. gate = tenant.layout.pipelines['gate'] FakeChange = namedtuple('FakeChange', ['project', 'branch']) fake_a = FakeChange(project1, 'master') fake_b = FakeChange(project2, 'master') with (pipeline_lock(self.zk_client, tenant.name, gate.name) as lock, self.createZKContext(lock) as ctx, gate.manager.currentContext(ctx)): gate.manager.getChangeQueue(fake_a, None) gate.manager.getChangeQueue(fake_b, None) q1 = gate.getQueue(project1.canonical_name, None) q2 = gate.getQueue(project2.canonical_name, None) self.assertEqual(q1.name, 'integrated') self.assertEqual(q2.name, 'integrated') @simple_layout("layouts/template-project-queue.yaml") def test_template_project_queue(self): "Test a shared queue can be constructed from a project-template" tenant = self.scheds.first.sched.abide.tenants.get('tenant-one') (trusted, project1) = tenant.getProject('org/project1') (trusted, project2) = tenant.getProject('org/project2') # Change queues are created lazy by the dependent pipeline manager # so retrieve the queue first without having to really enqueue a # change first. gate = tenant.layout.pipelines['gate'] FakeChange = namedtuple('FakeChange', ['project', 'branch']) fake_a = FakeChange(project1, 'master') fake_b = FakeChange(project2, 'master') with (pipeline_lock(self.zk_client, tenant.name, gate.name) as lock, self.createZKContext(lock) as ctx, gate.manager.currentContext(ctx)): gate.manager.getChangeQueue(fake_a, None) gate.manager.getChangeQueue(fake_b, None) q1 = gate.getQueue(project1.canonical_name, None) q2 = gate.getQueue(project2.canonical_name, None) self.assertEqual(q1.name, 'integrated') self.assertEqual(q2.name, 'integrated') @simple_layout("layouts/regex-template-queue.yaml") def test_regex_template_queue(self): "Test a shared queue can be constructed from a regex project-template" tenant = self.scheds.first.sched.abide.tenants.get('tenant-one') (trusted, project1) = tenant.getProject('org/project1') (trusted, project2) = tenant.getProject('org/project2') # Change queues are created lazy by the dependent pipeline manager # so retrieve the queue first without having to really enqueue a # change first. gate = tenant.layout.pipelines['gate'] FakeChange = namedtuple('FakeChange', ['project', 'branch']) fake_a = FakeChange(project1, 'master') fake_b = FakeChange(project2, 'master') with (pipeline_lock(self.zk_client, tenant.name, gate.name) as lock, self.createZKContext(lock) as ctx, gate.manager.currentContext(ctx)): gate.manager.getChangeQueue(fake_a, None) gate.manager.getChangeQueue(fake_b, None) q1 = gate.getQueue(project1.canonical_name, None) q2 = gate.getQueue(project2.canonical_name, None) self.assertEqual(q1.name, 'integrated') self.assertEqual(q2.name, 'integrated') @simple_layout("layouts/regex-queue.yaml") @skipIfMultiScheduler() def test_regex_queue(self): "Test a shared queue can be constructed from a regex project" tenant = self.scheds.first.sched.abide.tenants.get('tenant-one') (trusted, project1) = tenant.getProject('org/project1') (trusted, project2) = tenant.getProject('org/project2') # Change queues are created lazy by the dependent pipeline manager # so retrieve the queue first without having to really enqueue a # change first. gate = tenant.layout.pipelines['gate'] FakeChange = namedtuple('FakeChange', ['project', 'branch']) fake_a = FakeChange(project1, 'master') fake_b = FakeChange(project2, 'master') with (pipeline_lock(self.zk_client, tenant.name, gate.name) as lock, self.createZKContext(lock) as ctx, gate.manager.currentContext(ctx)): gate.manager.getChangeQueue(fake_a, None) gate.manager.getChangeQueue(fake_b, None) q1 = gate.getQueue(project1.canonical_name, None) q2 = gate.getQueue(project2.canonical_name, None) self.assertEqual(q1.name, 'integrated') self.assertEqual(q2.name, 'integrated') def test_queue_precedence(self): "Test that queue precedence works" self.hold_jobs_in_queue = True self.executor_server.hold_jobs_in_build = True A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) A.addApproval('Code-Review', 2) self.fake_gerrit.addEvent(A.addApproval('Approved', 1)) self.waitUntilSettled() self.hold_jobs_in_queue = False self.executor_api.release() self.waitUntilSettled() # Run one build at a time to ensure non-race order: self.orderedRelease() self.executor_server.hold_jobs_in_build = False self.waitUntilSettled() self.log.debug(self.history) self.assertEqual(self.history[0].pipeline, 'gate') self.assertEqual(self.history[1].pipeline, 'check') self.assertEqual(self.history[2].pipeline, 'gate') self.assertEqual(self.history[3].pipeline, 'gate') self.assertEqual(self.history[4].pipeline, 'check') self.assertEqual(self.history[5].pipeline, 'check') @simple_layout('layouts/two-check.yaml') def test_query_dependency_count(self): # Test that we efficiently query dependent changes A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B') B.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % ( B.subject, A.data['url']) self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1)) self.waitUntilSettled() # 1. The query to find the change id # from the Depends-On string "change:1" (simpleQuery) # 2. The query to populate the change once we know the id # (queryChange) self.assertEqual(A.queried, 2) self.assertEqual(B.queried, 1) def test_reconfigure_merge(self): """Test that two reconfigure events are merged""" # Wrap the recofiguration handler so we can count how many # times it runs. with mock.patch.object( zuul.scheduler.Scheduler, '_doTenantReconfigureEvent', wraps=self.scheds.first.sched._doTenantReconfigureEvent ) as mymock: with self.scheds.first.sched.run_handler_lock: self.create_branch('org/project', 'stable/diablo') self.fake_gerrit.addEvent( self.fake_gerrit.getFakeBranchCreatedEvent( 'org/project', 'stable/diablo')) self.create_branch('org/project', 'stable/essex') self.fake_gerrit.addEvent( self.fake_gerrit.getFakeBranchCreatedEvent( 'org/project', 'stable/essex')) for _ in iterate_timeout(60, 'jobs started'): if len(self.scheds.first.sched.trigger_events[ 'tenant-one']) == 2: break self.waitUntilSettled() mymock.assert_called_once() def test_live_reconfiguration(self): "Test that live reconfiguration works" self.executor_server.hold_jobs_in_build = True A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') A.addApproval('Code-Review', 2) self.fake_gerrit.addEvent(A.addApproval('Approved', 1)) self.waitUntilSettled() self.scheds.execute(lambda app: app.sched.reconfigure(app.config)) self.waitUntilSettled() self.executor_server.hold_jobs_in_build = False self.executor_server.release() self.waitUntilSettled() self.assertEqual(self.getJobFromHistory('project-merge').result, 'SUCCESS') self.assertEqual(self.getJobFromHistory('project-test1').result, 'SUCCESS') self.assertEqual(self.getJobFromHistory('project-test2').result, 'SUCCESS') self.assertEqual(A.data['status'], 'MERGED') self.assertEqual(A.reported, 2) def test_live_reconfiguration_layout_cache_fallback(self): # Test that re-calculating a dynamic fallback layout works after it # was removed during a reconfiguration. self.executor_server.hold_jobs_in_build = True in_repo_conf = textwrap.dedent( """ - job: name: project-test3 parent: project-test1 # add a job by the canonical project name - project: gate: jobs: - project-test3: dependencies: - project-merge """) file_dict = {'zuul.d/a.yaml': in_repo_conf} A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A', files=file_dict) A.addApproval('Code-Review', 2) self.fake_gerrit.addEvent(A.addApproval('Approved', 1)) self.waitUntilSettled() self.scheds.execute(lambda app: app.sched.reconfigure(app.config)) self.waitUntilSettled() tenant = self.scheds.first.sched.abide.tenants.get('tenant-one') pipeline = tenant.layout.pipelines['gate'] items = pipeline.getAllItems() self.assertEqual(len(items), 1) self.assertIsNone(items[0].layout_uuid) # Assert that the layout cache is empty after a reconfiguration. self.assertEqual(pipeline.manager._layout_cache, {}) B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B', parent='refs/changes/01/1/1') B.addApproval('Code-Review', 2) self.fake_gerrit.addEvent(B.addApproval('Approved', 1)) self.waitUntilSettled() items = pipeline.getAllItems() self.assertEqual(len(items), 2) for item in items: # Layout UUID should be set again for all live items. It had to # be re-calculated for the first item in the queue as it was reset # during re-enqueue. self.assertIsNotNone(item.layout_uuid) self.executor_server.hold_jobs_in_build = False self.executor_server.release() self.waitUntilSettled() self.assertEqual(A.data['status'], 'MERGED') self.assertEqual(A.reported, 2) self.assertEqual(B.data['status'], 'MERGED') self.assertEqual(B.reported, 2) self.assertHistory([ dict(name='project-merge', result='SUCCESS', changes='1,1'), dict(name='project-merge', result='SUCCESS', changes='1,1 2,1'), dict(name='project-test1', result='SUCCESS', changes='1,1'), dict(name='project-test1', result='SUCCESS', changes='1,1 2,1'), dict(name='project-test2', result='SUCCESS', changes='1,1'), dict(name='project-test2', result='SUCCESS', changes='1,1 2,1'), dict(name='project-test3', result='SUCCESS', changes='1,1'), dict(name='project-test3', result='SUCCESS', changes='1,1 2,1'), ], ordered=False) def test_live_reconfiguration_layout_cache_non_live(self): # Test that the layout UUID is only reset for live items. self.executor_server.hold_jobs_in_build = True in_repo_conf = textwrap.dedent( """ - job: name: project-test3 parent: project-test1 # add a job by the canonical project name - project: check: jobs: - project-test3: dependencies: - project-merge """) file_dict = {'zuul.d/a.yaml': in_repo_conf} A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A', files=file_dict) B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B', parent='refs/changes/01/1/1') B.setDependsOn(A, 1) self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1)) self.waitUntilSettled() self.scheds.execute(lambda app: app.sched.reconfigure(app.config)) self.waitUntilSettled() tenant = self.scheds.first.sched.abide.tenants.get('tenant-one') pipeline = tenant.layout.pipelines['check'] items = pipeline.getAllItems() self.assertEqual(len(items), 2) # Assert that the layout UUID of the live item is reset during a # reconfiguration, but non-live items keep their UUID. self.assertIsNotNone(items[0].layout_uuid) self.assertIsNone(items[1].layout_uuid) # Cache should be empty after a reconfiguration self.assertEqual(pipeline.manager._layout_cache, {}) self.executor_server.hold_jobs_in_build = False self.executor_server.release() self.waitUntilSettled() self.assertEqual(A.data['status'], 'NEW') self.assertEqual(A.reported, 0) self.assertEqual(B.data['status'], 'NEW') self.assertEqual(B.reported, 1) self.assertHistory([ dict(name='project-merge', result='SUCCESS', changes='1,1 2,1'), dict(name='project-test1', result='SUCCESS', changes='1,1 2,1'), dict(name='project-test2', result='SUCCESS', changes='1,1 2,1'), dict(name='project-test3', result='SUCCESS', changes='1,1 2,1'), ], ordered=False) def test_live_reconfiguration_command_socket(self): "Test that live reconfiguration via command socket works" # record previous tenant reconfiguration state, which may not be set old = self.scheds.first.sched.tenant_layout_state.get( 'tenant-one', EMPTY_LAYOUT_STATE) self.waitUntilSettled() command_socket = self.scheds.first.config.get( 'scheduler', 'command_socket') with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as s: s.connect(command_socket) s.sendall('full-reconfigure\n'.encode('utf8')) # Wait for full reconfiguration. Note that waitUntilSettled is not # reliable here because the reconfigure event may arrive in the # event queue after waitUntilSettled. start = time.time() while True: if time.time() - start > 15: raise Exception("Timeout waiting for full reconfiguration") new = self.scheds.first.sched.tenant_layout_state.get( 'tenant-one', EMPTY_LAYOUT_STATE) if old < new: break else: time.sleep(0) self.assertGreater(new.last_reconfigured, old.last_reconfigured) self.assertGreater(new.last_reconfigure_event_ltime, old.last_reconfigure_event_ltime) def test_tenant_reconfiguration_command_socket(self): "Test that single-tenant reconfiguration via command socket works" # record previous tenant reconfiguration state, which may not be set old = self.scheds.first.sched.tenant_layout_state.get( 'tenant-one', EMPTY_LAYOUT_STATE) self.waitUntilSettled() command_socket = self.scheds.first.config.get( 'scheduler', 'command_socket') with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as s: s.connect(command_socket) s.sendall('tenant-reconfigure ["tenant-one"]\n'.encode('utf8')) # Wait for full reconfiguration. Note that waitUntilSettled is not # reliable here because the reconfigure event may arrive in the # event queue after waitUntilSettled. start = time.time() while True: if time.time() - start > 15: raise Exception("Timeout waiting for full reconfiguration") new = self.scheds.first.sched.tenant_layout_state.get( 'tenant-one', EMPTY_LAYOUT_STATE) if old < new: break else: time.sleep(0) def test_double_live_reconfiguration_shared_queue(self): # This was a real-world regression. A change is added to # gate; a reconfigure happens, a second change which depends # on the first is added, and a second reconfiguration happens. # Ensure that both changes merge. # A failure may indicate incorrect caching or cleaning up of # references during a reconfiguration. self.executor_server.hold_jobs_in_build = True A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A') B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B') B.setDependsOn(A, 1) A.addApproval('Code-Review', 2) B.addApproval('Code-Review', 2) # Add the parent change. self.fake_gerrit.addEvent(A.addApproval('Approved', 1)) self.waitUntilSettled() self.executor_server.release('.*-merge') self.waitUntilSettled() # Reconfigure (with only one change in the pipeline). self.scheds.execute(lambda app: app.sched.reconfigure(app.config)) self.waitUntilSettled() # Add the child change. self.fake_gerrit.addEvent(B.addApproval('Approved', 1)) self.waitUntilSettled() self.executor_server.release('.*-merge') self.waitUntilSettled() # Reconfigure (with both in the pipeline). self.scheds.execute(lambda app: app.sched.reconfigure(app.config)) self.waitUntilSettled() self.executor_server.hold_jobs_in_build = False self.executor_server.release() self.waitUntilSettled() self.assertEqual(len(self.history), 8) self.assertEqual(A.data['status'], 'MERGED') self.assertEqual(A.reported, 2) self.assertEqual(B.data['status'], 'MERGED') self.assertEqual(B.reported, 2) def test_live_reconfiguration_del_project(self): # Test project deletion from tenant while changes are enqueued self.executor_server.hold_jobs_in_build = True A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B') C = self.fake_gerrit.addFakeChange('org/project1', 'master', 'C') self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1)) self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1)) self.waitUntilSettled() self.executor_server.release('.*-merge') self.waitUntilSettled() self.assertEqual(len(self.builds), 8) self.newTenantConfig('config/single-tenant/main-one-project.yaml') # This layout defines only org/project, not org/project1 self.commitConfigUpdate( 'common-config', 'layouts/live-reconfiguration-del-project.yaml') self.scheds.execute(lambda app: app.sched.reconfigure(app.config)) self.waitUntilSettled() self.executor_server.hold_jobs_in_build = False self.executor_server.release() self.waitUntilSettled() self.assertEqual(A.data['status'], 'NEW') self.assertEqual(B.data['status'], 'NEW') self.assertEqual(C.data['status'], 'NEW') self.assertEqual(A.reported, 1) self.assertEqual(B.reported, 0) self.assertEqual(C.reported, 0) self.assertHistory([ dict(name='project-merge', result='SUCCESS', changes='2,1'), dict(name='project-merge', result='SUCCESS', changes='1,1'), dict(name='project-merge', result='SUCCESS', changes='3,1'), dict(name='project-test1', result='ABORTED', changes='2,1'), dict(name='project-test2', result='ABORTED', changes='2,1'), dict(name='project1-project2-integration', result='ABORTED', changes='2,1'), dict(name='project-test1', result='ABORTED', changes='3,1'), dict(name='project-test2', result='ABORTED', changes='3,1'), dict(name='project1-project2-integration', result='ABORTED', changes='3,1'), dict(name='project-test1', result='SUCCESS', changes='1,1'), dict(name='project-test2', result='SUCCESS', changes='1,1'), ], ordered=False) tenant = self.scheds.first.sched.abide.tenants.get('tenant-one') self.assertEqual(len(tenant.layout.pipelines['check'].queues), 0) self.assertIn('Build succeeded', A.messages[0]) def test_live_reconfiguration_del_pipeline(self): # Test pipeline deletion while changes are enqueued self.executor_server.hold_jobs_in_build = True A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) self.waitUntilSettled() self.executor_server.release('.*-merge') self.waitUntilSettled() self.assertEqual(len(self.builds), 2) # This layout defines only org/project, not org/project1 self.commitConfigUpdate( 'common-config', 'layouts/live-reconfiguration-del-pipeline.yaml') self.scheds.execute(lambda app: app.sched.reconfigure(app.config)) self.waitUntilSettled() self.executor_server.hold_jobs_in_build = False self.executor_server.release() self.waitUntilSettled() self.assertEqual(A.data['status'], 'NEW') self.assertEqual(A.reported, 0) self.assertHistory([ dict(name='project-merge', result='SUCCESS', changes='1,1'), dict(name='project-test1', result='ABORTED', changes='1,1'), dict(name='project-test2', result='ABORTED', changes='1,1'), ], ordered=False) tenant = self.scheds.first.sched.abide.tenants.get('tenant-one') self.assertEqual(len(tenant.layout.pipelines), 0) def test_live_reconfiguration_del_tenant(self): # Test tenant deletion while changes are enqueued self.executor_server.hold_jobs_in_build = True A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B') C = self.fake_gerrit.addFakeChange('org/project1', 'master', 'C') self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1)) self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1)) self.waitUntilSettled() self.executor_server.release('.*-merge') self.waitUntilSettled() self.assertEqual(len(self.builds), 8) self.newTenantConfig('config/single-tenant/main-no-tenants.yaml') self.scheds.execute(lambda app: app.sched.reconfigure(app.config)) self.waitUntilSettled() self.executor_server.hold_jobs_in_build = False self.executor_server.release() self.waitUntilSettled() self.assertEqual(A.data['status'], 'NEW') self.assertEqual(B.data['status'], 'NEW') self.assertEqual(C.data['status'], 'NEW') self.assertEqual(A.reported, 0) self.assertEqual(B.reported, 0) self.assertEqual(C.reported, 0) self.assertHistory([ dict(name='project-merge', result='SUCCESS', changes='2,1'), dict(name='project-merge', result='SUCCESS', changes='1,1'), dict(name='project-merge', result='SUCCESS', changes='3,1'), dict(name='project-test1', result='ABORTED', changes='2,1'), dict(name='project-test2', result='ABORTED', changes='2,1'), dict(name='project1-project2-integration', result='ABORTED', changes='2,1'), dict(name='project-test1', result='ABORTED', changes='3,1'), dict(name='project-test2', result='ABORTED', changes='3,1'), dict(name='project1-project2-integration', result='ABORTED', changes='3,1'), dict(name='project-test1', result='ABORTED', changes='1,1'), dict(name='project-test2', result='ABORTED', changes='1,1'), ], ordered=False) tenant = self.scheds.first.sched.abide.tenants.get('tenant-one') self.assertIsNone(tenant) @simple_layout("layouts/reconfigure-failed-head.yaml") def test_live_reconfiguration_failed_change_at_head(self): # Test that if we reconfigure with a failed change at head, # that the change behind it isn't reparented onto it. self.executor_server.hold_jobs_in_build = True A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') A.addApproval('Code-Review', 2) B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B') B.addApproval('Code-Review', 2) self.executor_server.failJob('job1', A) self.fake_gerrit.addEvent(A.addApproval('Approved', 1)) self.fake_gerrit.addEvent(B.addApproval('Approved', 1)) self.waitUntilSettled() self.assertBuilds([ dict(name='job1', changes='1,1'), dict(name='job2', changes='1,1'), dict(name='job1', changes='1,1 2,1'), dict(name='job2', changes='1,1 2,1'), ]) self.release(self.builds[0]) self.waitUntilSettled() self.assertBuilds([ dict(name='job2', changes='1,1'), dict(name='job1', changes='2,1'), dict(name='job2', changes='2,1'), ]) # Unordered history comparison because the aborts can finish # in any order. self.assertHistory([ dict(name='job1', result='FAILURE', changes='1,1'), dict(name='job1', result='ABORTED', changes='1,1 2,1'), dict(name='job2', result='ABORTED', changes='1,1 2,1'), ], ordered=False) self.scheds.execute(lambda app: app.sched.reconfigure(app.config)) self.waitUntilSettled() self.executor_server.hold_jobs_in_build = False self.executor_server.release() self.waitUntilSettled() self.assertBuilds([]) self.assertHistory([ dict(name='job1', result='FAILURE', changes='1,1'), dict(name='job1', result='ABORTED', changes='1,1 2,1'), dict(name='job2', result='ABORTED', changes='1,1 2,1'), dict(name='job2', result='SUCCESS', changes='1,1'), dict(name='job1', result='SUCCESS', changes='2,1'), dict(name='job2', result='SUCCESS', changes='2,1'), ], ordered=False) self.assertEqual(A.data['status'], 'NEW') self.assertEqual(B.data['status'], 'MERGED') self.assertEqual(A.reported, 2) self.assertEqual(B.reported, 2) def test_delayed_repo_init(self): self.init_repo("org/new-project") files = {'README': ''} self.addCommitToRepo("org/new-project", 'Initial commit', files=files, tag='init') self.newTenantConfig('tenants/delayed-repo-init.yaml') self.commitConfigUpdate( 'common-config', 'layouts/delayed-repo-init.yaml') self.scheds.execute(lambda app: app.sched.reconfigure(app.config)) self.waitUntilSettled() A = self.fake_gerrit.addFakeChange('org/new-project', 'master', 'A') A.addApproval('Code-Review', 2) self.fake_gerrit.addEvent(A.addApproval('Approved', 1)) self.waitUntilSettled() self.assertEqual(self.getJobFromHistory('project-merge').result, 'SUCCESS') self.assertEqual(self.getJobFromHistory('project-test1').result, 'SUCCESS') self.assertEqual(self.getJobFromHistory('project-test2').result, 'SUCCESS') self.assertEqual(A.data['status'], 'MERGED') self.assertEqual(A.reported, 2) @simple_layout('layouts/single-job-with-nodeset.yaml') def test_live_reconfiguration_queued_node_requests(self): # Test that a job with a queued node request still has the # correct state after reconfiguration. self.fake_nodepool.pause() A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) self.waitUntilSettled() def get_job(): tenant = self.scheds.first.sched.abide.tenants['tenant-one'] for pipeline in tenant.layout.pipelines.values(): pipeline_status = pipeline.formatStatusJSON( self.scheds.first.sched.globals.websocket_url) for queue in pipeline_status['change_queues']: for head in queue['heads']: for item in head: for job in item['jobs']: if job['name'] == 'check-job': return job job = get_job() self.assertTrue(job['queued']) self.scheds.execute(lambda app: app.sched.reconfigure(self.config)) self.waitUntilSettled() job = get_job() self.assertTrue(job['queued']) self.fake_nodepool.unpause() self.waitUntilSettled() self.assertHistory([ dict(name='check-job', result='SUCCESS', changes='1,1'), ]) @simple_layout('layouts/trigger-sequence.yaml') def test_live_reconfiguration_trigger_sequence(self): # Test that events arriving after an event that triggers a # reconfiguration are handled after the reconfiguration # completes. in_repo_conf = "[{project: {tag: {jobs: [post-job]}}}]" file_dict = {'zuul.yaml': in_repo_conf} A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A', files=file_dict) sched = self.scheds.first.sched # Hold the management queue so that we don't process any # reconfiguration events yet. with management_queue_lock( self.zk_client, 'tenant-one', blocking=False ): with sched.run_handler_lock: A.setMerged() # Submit two events while no processing is happening: # A change merged event that will trigger a reconfiguration self.fake_gerrit.addEvent(A.getChangeMergedEvent()) # And a tag event which should only run a job after # the config change above is in effect. event = self.fake_gerrit.addFakeTag( 'org/project', 'master', 'foo') self.fake_gerrit.addEvent(event) # Wait for the tenant trigger queue to empty out, and for # us to have a tenant management as well as a pipeline # trigger event. At this point, we should be deferring # the trigger event until the management event is handled. for _ in iterate_timeout(60, 'queues'): with sched.run_handler_lock: if sched.trigger_events['tenant-one'].hasEvents(): continue if not sched.pipeline_trigger_events[ 'tenant-one']['tag'].hasEvents(): continue if not sched.management_events['tenant-one'].hasEvents(): continue break # Now we can resume and process the reconfiguration event sched.wake_event.set() self.waitUntilSettled() self.assertHistory([ dict(name='post-job', result='SUCCESS'), ]) @simple_layout('layouts/repo-deleted.yaml') def test_repo_deleted(self): self.init_repo("org/delete-project") A = self.fake_gerrit.addFakeChange('org/delete-project', 'master', 'A') A.addApproval('Code-Review', 2) self.fake_gerrit.addEvent(A.addApproval('Approved', 1)) self.waitUntilSettled() self.assertEqual(self.getJobFromHistory('project-merge').result, 'SUCCESS') self.assertEqual(self.getJobFromHistory('project-test1').result, 'SUCCESS') self.assertEqual(self.getJobFromHistory('project-test2').result, 'SUCCESS') self.assertEqual(A.data['status'], 'MERGED') self.assertEqual(A.reported, 2) # Delete org/new-project zuul repo. Should be recloned. p = 'org/delete-project' if os.path.exists(os.path.join(self.merger_src_root, p)): shutil.rmtree(os.path.join(self.merger_src_root, p)) if os.path.exists(os.path.join(self.executor_src_root, p)): shutil.rmtree(os.path.join(self.executor_src_root, p)) B = self.fake_gerrit.addFakeChange('org/delete-project', 'master', 'B') B.addApproval('Code-Review', 2) self.fake_gerrit.addEvent(B.addApproval('Approved', 1)) self.waitUntilSettled() self.assertEqual(self.getJobFromHistory('project-merge').result, 'SUCCESS') self.assertEqual(self.getJobFromHistory('project-test1').result, 'SUCCESS') self.assertEqual(self.getJobFromHistory('project-test2').result, 'SUCCESS') self.assertEqual(B.data['status'], 'MERGED') self.assertEqual(B.reported, 2) @simple_layout('layouts/untrusted-secrets.yaml') def test_untrusted_secrets(self): "Test untrusted secrets" A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A') self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) self.waitUntilSettled() self.assertHistory([]) self.assertEqual(A.patchsets[0]['approvals'][0]['value'], "-1") self.assertIn('does not allow post-review job', A.messages[0]) @simple_layout('layouts/tags.yaml') def test_tags(self): "Test job tags" A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A') B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B') self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1)) self.waitUntilSettled() self.assertEqual(len(self.history), 2) results = {self.getJobFromHistory('merge', project='org/project1').uuid: ['extratag', 'merge'], self.getJobFromHistory('merge', project='org/project2').uuid: ['merge']} for build in self.history: self.assertEqual(results.get(build.uuid, ''), build.parameters['zuul'].get('jobtags')) def test_timer_template(self): "Test that a periodic job is triggered" # This test can not use simple_layout because it must start # with a configuration which does not include a # timer-triggered job so that we have an opportunity to set # the hold flag before the first job. self.create_branch('org/project', 'stable') self.fake_gerrit.addEvent( self.fake_gerrit.getFakeBranchCreatedEvent( 'org/project', 'stable')) self.executor_server.hold_jobs_in_build = True self.commitConfigUpdate('common-config', 'layouts/timer-template.yaml') self.scheds.execute(lambda app: app.sched.reconfigure(app.config)) # The pipeline triggers every second, so we should have seen # several by now. time.sleep(5) self.waitUntilSettled() self.assertEqual(len(self.builds), 2) merge_count_project1 = 0 for job in self.merge_job_history.get( zuul.model.MergeRequest.REF_STATE ): if job.payload["items"][0]["project"] == "org/project1": merge_count_project1 += 1 self.assertEquals(merge_count_project1, 0, "project1 shouldn't have any refstate call") self.executor_server.hold_jobs_in_build = False # Stop queuing timer triggered jobs so that the assertions # below don't race against more jobs being queued. self.commitConfigUpdate('common-config', 'layouts/no-timer.yaml') self.scheds.execute(lambda app: app.sched.reconfigure(app.config)) self.waitUntilSettled() # If APScheduler is in mid-event when we remove the job, we # can end up with one more event firing, so give it an extra # second to settle. time.sleep(1) self.waitUntilSettled() self.executor_server.release() self.waitUntilSettled() self.assertHistory([ dict(name='project-bitrot', result='SUCCESS', ref='refs/heads/master'), dict(name='project-bitrot', result='SUCCESS', ref='refs/heads/stable'), ], ordered=False) def _test_timer(self, config_file): # This test can not use simple_layout because it must start # with a configuration which does not include a # timer-triggered job so that we have an opportunity to set # the hold flag before the first job. timer = self.scheds.first.sched.connections.drivers['timer'] start_jobs = timer.apsched.get_jobs() self.create_branch('org/project', 'stable') self.fake_gerrit.addEvent( self.fake_gerrit.getFakeBranchCreatedEvent( 'org/project', 'stable')) self.executor_server.hold_jobs_in_build = True self.commitConfigUpdate('common-config', config_file) self.scheds.execute(lambda app: app.sched.reconfigure(app.config)) first_jobs = timer.apsched.get_jobs() # Collect the currently cached branches in order to later check, # that the timer driver refreshes the cache. cached_versions = {} tenant = self.scheds.first.sched.abide.tenants['tenant-one'] for project_name in tenant.layout.project_configs: _, project = tenant.getProject('org/project') for branch in project.source.getProjectBranches(project, tenant): event = self._create_dummy_event(project, branch) change_key = project.source.getChangeKey(event) change = project.source.getChange(change_key, event=event) cached_versions[branch] = change.cache_version # The pipeline triggers every second, so we should have seen # several by now. for _ in iterate_timeout(60, 'jobs started'): if len(self.builds) > 1: break # Ensure that the status json has the ref so we can render it in the # web ui. pipeline = self.scheds.first.sched.abide.tenants[ 'tenant-one'].layout.pipelines['periodic'] pipeline_status = pipeline.formatStatusJSON( self.scheds.first.sched.globals.websocket_url) first = pipeline_status['change_queues'][0]['heads'][0][0] second = pipeline_status['change_queues'][1]['heads'][0][0] self.assertIn(first['changes'][0]['ref'], ['refs/heads/master', 'refs/heads/stable']) self.assertIn(second['changes'][0]['ref'], ['refs/heads/master', 'refs/heads/stable']) self.executor_server.hold_jobs_in_build = False # Stop queuing timer triggered jobs so that the assertions # below don't race against more jobs being queued. self.commitConfigUpdate('common-config', 'layouts/no-timer.yaml') self.scheds.execute(lambda app: app.sched.reconfigure(app.config)) second_jobs = timer.apsched.get_jobs() self.waitUntilSettled() # If APScheduler is in mid-event when we remove the job, we # can end up with one more event firing, so give it an extra # second to settle. time.sleep(3) self.waitUntilSettled() self.executor_server.release() self.waitUntilSettled() self.assertTrue(len(self.history) > 1) for job in self.history[:1]: self.assertEqual(job.result, 'SUCCESS') self.assertEqual(job.name, 'project-bitrot') self.assertIn(job.ref, ('refs/heads/stable', 'refs/heads/master')) for project_name in tenant.layout.project_configs: _, project = tenant.getProject('org/project') for branch in project.source.getProjectBranches(project, tenant): event = self._create_dummy_event(project, branch) change_key = project.source.getChangeKey(event) change = project.source.getChange(change_key, event=event) # Make sure the timer driver refreshed the cache self.assertGreater(change.cache_version, cached_versions[branch]) # We start with no jobs, and our first reconfigure should add jobs self.assertTrue(len(first_jobs) > len(start_jobs)) # Our second reconfigure should return us to no jobs self.assertEqual(start_jobs, second_jobs) def _create_dummy_event(self, project, branch): event = zuul.model.TriggerEvent() event.type = 'test' event.project_hostname = project.canonical_hostname event.project_name = project.name event.ref = f'refs/heads/{branch}' event.branch = branch event.zuul_event_id = str(uuid4().hex) event.timestamp = time.time() return event def test_timer(self): "Test that a periodic job is triggered" self._test_timer('layouts/timer.yaml') def test_timer_with_jitter(self): "Test that a periodic job with a jitter is triggered" self._test_timer('layouts/timer-jitter.yaml') @simple_layout('layouts/timer-jitter.yaml') def test_timer_preserve_jobs(self): # This tests that we keep the same apsched jobs if possible # when reconfiguring. If a reconfiguration happens during the # "jitter" period, we might end up not running jobs unless we # preserve the exact job object across reconfiguration. timer = self.scheds.first.sched.connections.drivers['timer'] old_jobs = timer.apsched.get_jobs() self.scheds.execute(lambda app: app.sched.reconfigure(app.config)) new_jobs = timer.apsched.get_jobs() self.assertEqual(old_jobs, new_jobs) def test_idle(self): "Test that frequent periodic jobs work" # This test can not use simple_layout because it must start # with a configuration which does not include a # timer-triggered job so that we have an opportunity to set # the hold flag before the first job. self.executor_server.hold_jobs_in_build = True for x in range(1, 3): # Test that timer triggers periodic jobs even across # layout config reloads. # Start timer trigger self.commitConfigUpdate('common-config', 'layouts/idle.yaml') self.scheds.execute(lambda app: app.sched.reconfigure(app.config)) self.waitUntilSettled() # The pipeline triggers every second, so we should have seen # several by now. time.sleep(5) # Stop queuing timer triggered jobs so that the assertions # below don't race against more jobs being queued. self.commitConfigUpdate('common-config', 'layouts/no-timer.yaml') self.scheds.execute(lambda app: app.sched.reconfigure(app.config)) self.waitUntilSettled() # If APScheduler is in mid-event when we remove the job, # we can end up with one more event firing, so give it an # extra second to settle. time.sleep(1) self.waitUntilSettled() self.assertEqual(len(self.builds), 1, 'Timer builds iteration #%d' % x) self.executor_server.release('.*') self.waitUntilSettled() self.assertEqual(len(self.builds), 0) self.assertEqual(len(self.history), x) @simple_layout('layouts/smtp.yaml') def test_check_smtp_pool(self): A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') self.waitUntilSettled() self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) self.waitUntilSettled() self.assertEqual(len(self.smtp_messages), 2) # A.messages only holds what FakeGerrit places in it. Thus we # work on the knowledge of what the first message should be as # it is only configured to go to SMTP. self.assertEqual('zuul@example.com', self.smtp_messages[0]['from_email']) self.assertEqual(['you@example.com'], self.smtp_messages[0]['to_email']) self.assertEqual('Starting check jobs.', self.smtp_messages[0]['body']) self.assertEqual('zuul_from@example.com', self.smtp_messages[1]['from_email']) self.assertEqual(['alternative_me@example.com'], self.smtp_messages[1]['to_email']) self.assertEqual(A.messages[0], self.smtp_messages[1]['body']) @simple_layout('layouts/smtp.yaml') @mock.patch('zuul.driver.gerrit.gerritreporter.GerritReporter.report') def test_failed_reporter(self, report_mock): '''Test that one failed reporter doesn't break other reporters''' # Warning hacks. We sort the reports here so that the test is # deterministic. Gerrit reporting will fail, but smtp reporting # should succeed. report_mock.side_effect = Exception('Gerrit failed to report') tenant = self.scheds.first.sched.abide.tenants.get('tenant-one') check = tenant.layout.pipelines['check'] check.success_actions = sorted(check.success_actions, key=lambda x: x.name) self.assertEqual(check.success_actions[0].name, 'gerrit') self.assertEqual(check.success_actions[1].name, 'smtp') A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') self.waitUntilSettled() self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) self.waitUntilSettled() # We know that if gerrit ran first and failed and smtp ran second # and sends mail then we handle failures in reporters gracefully. self.assertEqual(len(self.smtp_messages), 2) # A.messages only holds what FakeGerrit places in it. Thus we # work on the knowledge of what the first message should be as # it is only configured to go to SMTP. self.assertEqual('zuul@example.com', self.smtp_messages[0]['from_email']) self.assertEqual(['you@example.com'], self.smtp_messages[0]['to_email']) self.assertEqual('Starting check jobs.', self.smtp_messages[0]['body']) self.assertEqual('zuul_from@example.com', self.smtp_messages[1]['from_email']) self.assertEqual(['alternative_me@example.com'], self.smtp_messages[1]['to_email']) # This double checks that Gerrit side failed self.assertEqual(A.messages, []) def test_timer_smtp(self): "Test that a periodic job is triggered" # This test can not use simple_layout because it must start # with a configuration which does not include a # timer-triggered job so that we have an opportunity to set # the hold flag before the first job. self.executor_server.hold_jobs_in_build = True self.commitConfigUpdate('common-config', 'layouts/timer-smtp.yaml') self.scheds.execute(lambda app: app.sched.reconfigure(app.config)) # The pipeline triggers every second, so we should have seen # several by now. time.sleep(5) self.waitUntilSettled() self.assertEqual(len(self.builds), 2) self.executor_server.release('.*') self.waitUntilSettled() self.assertEqual(len(self.history), 2) self.assertEqual(self.getJobFromHistory( 'project-bitrot-stable-old').result, 'SUCCESS') self.assertEqual(self.getJobFromHistory( 'project-bitrot-stable-older').result, 'SUCCESS') self.assertEqual(len(self.smtp_messages), 1) # A.messages only holds what FakeGerrit places in it. Thus we # work on the knowledge of what the first message should be as # it is only configured to go to SMTP. self.assertEqual('zuul_from@example.com', self.smtp_messages[0]['from_email']) self.assertEqual(['alternative_me@example.com'], self.smtp_messages[0]['to_email']) self.assertIn('Subject: Periodic check for org/project succeeded', self.smtp_messages[0]['headers']) # Stop queuing timer triggered jobs and let any that may have # queued through so that end of test assertions pass. self.commitConfigUpdate('common-config', 'layouts/no-timer.yaml') self.scheds.execute(lambda app: app.sched.reconfigure(app.config)) self.waitUntilSettled() # If APScheduler is in mid-event when we remove the job, we # can end up with one more event firing, so give it an extra # second to settle. time.sleep(1) self.waitUntilSettled() self.executor_server.release('.*') self.waitUntilSettled() @skip("Disabled for early v3 development") def test_timer_sshkey(self): "Test that a periodic job can setup SSH key authentication" self.worker.hold_jobs_in_build = True self.config.set('zuul', 'layout_config', 'tests/fixtures/layout-timer.yaml') self.scheds.execute(lambda app: app.sched.reconfigure(app.config)) self.registerJobs() # The pipeline triggers every second, so we should have seen # several by now. time.sleep(5) self.waitUntilSettled() self.assertEqual(len(self.builds), 2) ssh_wrapper = os.path.join(self.git_root, ".ssh_wrapper_gerrit") self.assertTrue(os.path.isfile(ssh_wrapper)) with open(ssh_wrapper) as f: ssh_wrapper_content = f.read() self.assertIn("fake_id_rsa", ssh_wrapper_content) # In the unit tests Merger runs in the same process, # so we see its' environment variables self.assertEqual(os.environ['GIT_SSH'], ssh_wrapper) self.worker.release('.*') self.waitUntilSettled() self.assertEqual(len(self.history), 2) self.assertEqual(self.getJobFromHistory( 'project-bitrot-stable-old').result, 'SUCCESS') self.assertEqual(self.getJobFromHistory( 'project-bitrot-stable-older').result, 'SUCCESS') # Stop queuing timer triggered jobs and let any that may have # queued through so that end of test assertions pass. self.config.set('zuul', 'layout_config', 'tests/fixtures/layout-no-timer.yaml') self.scheds.execute(lambda app: app.sched.reconfigure(app.config)) self.registerJobs() self.waitUntilSettled() # If APScheduler is in mid-event when we remove the job, we # can end up with one more event firing, so give it an extra # second to settle. time.sleep(1) self.waitUntilSettled() self.worker.release('.*') self.waitUntilSettled() @simple_layout('layouts/rate-limit.yaml') def test_queue_rate_limiting(self): "Test that DependentPipelines are rate limited with dep across window" self.executor_server.hold_jobs_in_build = True A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B') C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C') C.setDependsOn(B, 1) self.executor_server.failJob('project-test1', A) A.addApproval('Code-Review', 2) B.addApproval('Code-Review', 2) C.addApproval('Code-Review', 2) self.fake_gerrit.addEvent(A.addApproval('Approved', 1)) self.fake_gerrit.addEvent(B.addApproval('Approved', 1)) self.fake_gerrit.addEvent(C.addApproval('Approved', 1)) self.waitUntilSettled() # Only A and B will have their merge jobs queued because # window is 2. self.assertEqual(len(self.builds), 2) self.assertEqual(self.builds[0].name, 'project-merge') self.assertEqual(self.builds[1].name, 'project-merge') # Release the merge jobs one at a time. self.builds[0].release() self.waitUntilSettled() self.builds[0].release() self.waitUntilSettled() # Only A and B will have their test jobs queued because # window is 2. self.assertEqual(len(self.builds), 4) self.assertEqual(self.builds[0].name, 'project-test1') self.assertEqual(self.builds[1].name, 'project-test2') self.assertEqual(self.builds[2].name, 'project-test1') self.assertEqual(self.builds[3].name, 'project-test2') self.executor_server.release('project-.*') self.waitUntilSettled() tenant = self.scheds.first.sched.abide.tenants.get('tenant-one') queue = tenant.layout.pipelines['gate'].queues[0] # A failed so window is reduced by 1 to 1. self.assertEqual(queue.window, 1) self.assertEqual(queue.window_floor, 1) self.assertEqual(queue.window_ceiling, 2) self.assertEqual(A.data['status'], 'NEW') # Gate is reset and only B's merge job is queued because # window shrunk to 1. self.assertEqual(len(self.builds), 1) self.assertEqual(self.builds[0].name, 'project-merge') self.executor_server.release('.*-merge') self.waitUntilSettled() # Only B's test jobs are queued because window is still 1. self.assertEqual(len(self.builds), 2) self.assertEqual(self.builds[0].name, 'project-test1') self.assertEqual(self.builds[1].name, 'project-test2') self.executor_server.release('project-.*') self.waitUntilSettled() # B was successfully merged so window is increased to 2. self.assertEqual(queue.window, 2) self.assertEqual(queue.window_floor, 1) self.assertEqual(queue.window_ceiling, 2) self.assertEqual(B.data['status'], 'MERGED') # Only C is left and its merge job is queued. self.assertEqual(len(self.builds), 1) self.assertEqual(self.builds[0].name, 'project-merge') self.executor_server.release('.*-merge') self.waitUntilSettled() # After successful merge job the test jobs for C are queued. self.assertEqual(len(self.builds), 2) self.assertEqual(self.builds[0].name, 'project-test1') self.assertEqual(self.builds[1].name, 'project-test2') self.executor_server.release('project-.*') self.waitUntilSettled() # C successfully merged but hit the ceiling of 2 self.assertEqual(queue.window, 2) self.assertEqual(queue.window_floor, 1) self.assertEqual(queue.window_ceiling, 2) self.assertEqual(C.data['status'], 'MERGED') @simple_layout('layouts/rate-limit.yaml') def test_queue_rate_limiting_dependent(self): "Test that DependentPipelines are rate limited with dep in window" self.executor_server.hold_jobs_in_build = True A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B') C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C') B.setDependsOn(A, 1) self.executor_server.failJob('project-test1', A) A.addApproval('Code-Review', 2) B.addApproval('Code-Review', 2) C.addApproval('Code-Review', 2) self.fake_gerrit.addEvent(A.addApproval('Approved', 1)) self.fake_gerrit.addEvent(B.addApproval('Approved', 1)) self.fake_gerrit.addEvent(C.addApproval('Approved', 1)) self.waitUntilSettled() # Only A and B will have their merge jobs queued because # window is 2. self.assertEqual(len(self.builds), 2) self.assertEqual(self.builds[0].name, 'project-merge') self.assertEqual(self.builds[1].name, 'project-merge') self.orderedRelease(2) # Only A and B will have their test jobs queued because # window is 2. self.assertEqual(len(self.builds), 4) self.assertEqual(self.builds[0].name, 'project-test1') self.assertEqual(self.builds[1].name, 'project-test2') self.assertEqual(self.builds[2].name, 'project-test1') self.assertEqual(self.builds[3].name, 'project-test2') self.executor_server.release('project-.*') self.waitUntilSettled() tenant = self.scheds.first.sched.abide.tenants.get('tenant-one') queue = tenant.layout.pipelines['gate'].queues[0] # A failed so window is reduced by 1 to 1. self.assertEqual(queue.window, 1) self.assertEqual(queue.window_floor, 1) self.assertEqual(A.data['status'], 'NEW') self.assertEqual(B.data['status'], 'NEW') # Gate is reset and only C's merge job is queued because # window shrunk to 1 and A and B were dequeued. self.assertEqual(len(self.builds), 1) self.assertEqual(self.builds[0].name, 'project-merge') self.orderedRelease(1) # Only C's test jobs are queued because window is still 1. self.assertEqual(len(self.builds), 2) builds = self.getSortedBuilds() self.assertEqual(builds[0].name, 'project-test1') self.assertEqual(builds[1].name, 'project-test2') self.executor_server.release('project-.*') self.waitUntilSettled() # C was successfully merged so window is increased to 2. self.assertEqual(queue.window, 2) self.assertEqual(queue.window_floor, 1) self.assertEqual(C.data['status'], 'MERGED') @simple_layout('layouts/rate-limit-reconfigure.yaml') def test_queue_rate_limiting_reconfigure(self): """Test that changes survive a reconfigure when no longer in window. This is a regression tests for a case that lead to an exception during re-enqueue. The exception happened when former active items had already build results but then dropped out of the active window. During re-enqueue the job graph was not re-initialized because the items were no longer active. """ self.executor_server.hold_jobs_in_build = True A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B') C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C') D = self.fake_gerrit.addFakeChange('org/project', 'master', 'D') A.addApproval('Code-Review', 2) B.addApproval('Code-Review', 2) C.addApproval('Code-Review', 2) D.addApproval('Code-Review', 2) self.fake_gerrit.addEvent(A.addApproval('Approved', 1)) self.fake_gerrit.addEvent(B.addApproval('Approved', 1)) self.fake_gerrit.addEvent(C.addApproval('Approved', 1)) self.fake_gerrit.addEvent(D.addApproval('Approved', 1)) self.waitUntilSettled() self.assertEqual(len(self.builds), 4) self.assertEqual(self.builds[0].name, 'project-merge') self.assertEqual(self.builds[1].name, 'project-merge') self.assertEqual(self.builds[2].name, 'project-merge') self.assertEqual(self.builds[3].name, 'project-merge') self.orderedRelease(4) self.assertEqual(len(self.builds), 8) self.assertEqual(self.builds[0].name, 'project-test1') self.assertEqual(self.builds[1].name, 'project-test2') self.assertEqual(self.builds[2].name, 'project-test1') self.assertEqual(self.builds[3].name, 'project-test2') self.assertEqual(self.builds[4].name, 'project-test1') self.assertEqual(self.builds[5].name, 'project-test2') self.assertEqual(self.builds[6].name, 'project-test1') self.assertEqual(self.builds[7].name, 'project-test2') self.executor_server.failJob('project-test1', B) self.builds[2].release() self.builds[3].release() self.waitUntilSettled() self.assertEqual(len(self.builds), 4) # A's jobs self.assertEqual(self.builds[0].name, 'project-test1') self.assertEqual(self.builds[1].name, 'project-test2') # C's and D's merge jobs self.assertEqual(self.builds[2].name, 'project-merge') self.assertEqual(self.builds[3].name, 'project-merge') # Release merge jobs of C, D after speculative gate reset self.executor_server.release('project-merge') self.waitUntilSettled() self.assertEqual(len(self.builds), 6) # A's jobs self.assertEqual(self.builds[0].name, 'project-test1') self.assertEqual(self.builds[1].name, 'project-test2') # C's + D's jobs self.assertEqual(self.builds[2].name, 'project-test1') self.assertEqual(self.builds[3].name, 'project-test2') self.assertEqual(self.builds[4].name, 'project-test1') self.assertEqual(self.builds[5].name, 'project-test2') # Fail D's job so we have a build results for an item that # is not in the active window after B is reported # (condition that previously lead to an exception) self.executor_server.failJob('project-test1', D) self.builds[4].release() self.waitUntilSettled() # Release A's jobs self.builds[0].release() self.builds[1].release() self.waitUntilSettled() self.assertEqual(len(self.builds), 3) # C's jobs self.assertEqual(self.builds[0].name, 'project-test1') self.assertEqual(self.builds[1].name, 'project-test2') # D's remaining job self.assertEqual(self.builds[2].name, 'project-test2') tenant = self.scheds.first.sched.abide.tenants.get('tenant-one') queue = tenant.layout.pipelines['gate'].queues[0] self.assertEqual(queue.window, 1) # D dropped out of the window self.assertFalse(queue.queue[-1].active) self.commitConfigUpdate('org/common-config', 'layouts/rate-limit-reconfigure2.yaml') self.scheds.execute(lambda app: app.sched.reconfigure(app.config)) self.waitUntilSettled() # D's remaining job should still be queued self.assertEqual(len(self.builds), 3) self.executor_server.release('project-.*') self.waitUntilSettled() @simple_layout('layouts/reconfigure-window.yaml') def test_reconfigure_window_shrink(self): # Test the active window shrinking during reconfiguration self.executor_server.hold_jobs_in_build = True A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') A.addApproval('Code-Review', 2) self.fake_gerrit.addEvent(A.addApproval('Approved', 1)) self.waitUntilSettled() B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B') B.addApproval('Code-Review', 2) self.fake_gerrit.addEvent(B.addApproval('Approved', 1)) self.waitUntilSettled() tenant = self.scheds.first.sched.abide.tenants.get('tenant-one') queue = tenant.layout.pipelines['gate'].queues[0] self.assertEqual(queue.window, 20) self.assertTrue(len(self.builds), 4) self.executor_server.release('job1') self.waitUntilSettled() self.commitConfigUpdate('org/common-config', 'layouts/reconfigure-window2.yaml') self.scheds.execute(lambda app: app.sched.reconfigure(app.config)) self.waitUntilSettled() tenant = self.scheds.first.sched.abide.tenants.get('tenant-one') queue = tenant.layout.pipelines['gate'].queues[0] # Even though we have configured a smaller window, the value # on the existing shared queue should be used. self.assertEqual(queue.window, 20) self.assertTrue(len(self.builds), 4) self.scheds.execute(lambda app: app.sched.reconfigure(app.config)) self.waitUntilSettled() tenant = self.scheds.first.sched.abide.tenants.get('tenant-one') queue = tenant.layout.pipelines['gate'].queues[0] self.assertEqual(queue.window, 20) self.assertTrue(len(self.builds), 4) self.executor_server.hold_jobs_in_build = False self.executor_server.release() self.waitUntilSettled() self.assertHistory([ dict(name='job1', result='SUCCESS', changes='1,1'), dict(name='job1', result='SUCCESS', changes='1,1 2,1'), dict(name='job2', result='SUCCESS', changes='1,1'), dict(name='job2', result='SUCCESS', changes='1,1 2,1'), ], ordered=False) @simple_layout('layouts/reconfigure-window-fixed.yaml') def test_reconfigure_window_fixed(self): # Test the active window shrinking during reconfiguration self.executor_server.hold_jobs_in_build = True A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') A.addApproval('Code-Review', 2) self.fake_gerrit.addEvent(A.addApproval('Approved', 1)) self.waitUntilSettled() B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B') B.addApproval('Code-Review', 2) self.fake_gerrit.addEvent(B.addApproval('Approved', 1)) self.waitUntilSettled() tenant = self.scheds.first.sched.abide.tenants.get('tenant-one') queue = tenant.layout.pipelines['gate'].queues[0] self.assertEqual(queue.window, 2) self.assertEqual(len(self.builds), 4) self.waitUntilSettled() self.commitConfigUpdate('org/common-config', 'layouts/reconfigure-window-fixed2.yaml') self.scheds.execute(lambda app: app.sched.reconfigure(app.config)) self.waitUntilSettled() tenant = self.scheds.first.sched.abide.tenants.get('tenant-one') queue = tenant.layout.pipelines['gate'].queues[0] # Because we have configured a static window, it should # be allowed to shrink on reconfiguration. self.assertEqual(queue.window, 1) # B is outside the window, but still marked active until the # next pass through the queue processor. self.assertEqual(len(self.builds), 4) self.scheds.execute(lambda app: app.sched.reconfigure(app.config)) self.waitUntilSettled() tenant = self.scheds.first.sched.abide.tenants.get('tenant-one') queue = tenant.layout.pipelines['gate'].queues[0] self.assertEqual(queue.window, 1) self.waitUntilSettled() # B's builds should not be canceled self.assertEqual(len(self.builds), 4) self.executor_server.hold_jobs_in_build = False self.executor_server.release() self.waitUntilSettled() self.assertHistory([ dict(name='job1', result='SUCCESS', changes='1,1'), dict(name='job2', result='SUCCESS', changes='1,1'), dict(name='job1', result='SUCCESS', changes='1,1 2,1'), dict(name='job2', result='SUCCESS', changes='1,1 2,1'), ], ordered=False) @simple_layout('layouts/reconfigure-window-fixed.yaml') def test_reconfigure_window_fixed_requests(self): # Test the active window shrinking during reconfiguration with # outstanding node requests self.executor_server.hold_jobs_in_build = True # Start the jobs for A A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') A.addApproval('Code-Review', 2) self.fake_gerrit.addEvent(A.addApproval('Approved', 1)) self.waitUntilSettled() self.log.debug("A complete") # Hold node requests for B self.fake_nodepool.pause() B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B') B.addApproval('Code-Review', 2) self.fake_gerrit.addEvent(B.addApproval('Approved', 1)) self.waitUntilSettled() self.log.debug("B complete") tenant = self.scheds.first.sched.abide.tenants.get('tenant-one') queue = tenant.layout.pipelines['gate'].queues[0] self.assertEqual(queue.window, 2) self.assertEqual(len(self.builds), 2) self.waitUntilSettled() self.commitConfigUpdate('org/common-config', 'layouts/reconfigure-window-fixed2.yaml') self.scheds.execute(lambda app: app.sched.reconfigure(app.config)) self.waitUntilSettled() self.log.debug("Reconfiguration complete") tenant = self.scheds.first.sched.abide.tenants.get('tenant-one') queue = tenant.layout.pipelines['gate'].queues[0] # Because we have configured a static window, it should # be allowed to shrink on reconfiguration. self.assertEqual(queue.window, 1) self.assertEqual(len(self.builds), 2) # After the previous reconfig, the queue processor will have # run and marked B inactive; run another reconfiguration so # that we're testing what happens when we reconfigure after # the active window having shrunk. self.scheds.execute(lambda app: app.sched.reconfigure(app.config)) # Unpause the node requests now self.fake_nodepool.unpause() self.waitUntilSettled() self.log.debug("Nodepool unpause complete") # Allow A to merge and B to enter the active window and complete self.executor_server.hold_jobs_in_build = False self.executor_server.release() self.waitUntilSettled() self.log.debug("Executor unpause complete") tenant = self.scheds.first.sched.abide.tenants.get('tenant-one') queue = tenant.layout.pipelines['gate'].queues[0] self.assertEqual(queue.window, 1) self.waitUntilSettled() self.assertHistory([ dict(name='job1', result='SUCCESS', changes='1,1'), dict(name='job2', result='SUCCESS', changes='1,1'), dict(name='job1', result='SUCCESS', changes='1,1 2,1'), dict(name='job2', result='SUCCESS', changes='1,1 2,1'), ], ordered=False) @simple_layout('layouts/footer-message.yaml') def test_footer_message(self): "Test a pipeline's footer message is correctly added to the report." A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') A.addApproval('Code-Review', 2) self.executor_server.failJob('project-test1', A) self.fake_gerrit.addEvent(A.addApproval('Approved', 1)) self.waitUntilSettled() B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B') B.addApproval('Code-Review', 2) self.fake_gerrit.addEvent(B.addApproval('Approved', 1)) self.waitUntilSettled() self.assertEqual(2, len(self.smtp_messages)) failure_msg = """\ Build failed. For information on how to proceed, see \ http://wiki.example.org/Test_Failures""" footer_msg = """\ For CI problems and help debugging, contact ci@example.org""" self.assertTrue(self.smtp_messages[0]['body'].startswith(failure_msg)) self.assertTrue(self.smtp_messages[0]['body'].endswith(footer_msg)) self.assertFalse(self.smtp_messages[1]['body'].startswith(failure_msg)) self.assertTrue(self.smtp_messages[1]['body'].endswith(footer_msg)) @simple_layout('layouts/start-message.yaml') def test_start_message(self): "Test a pipeline's start message is correctly added to the report." A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') A.addApproval('Code-Review', 2) self.fake_gerrit.addEvent(A.addApproval('Approved', 1)) self.waitUntilSettled() self.assertEqual(1, len(self.smtp_messages)) start_msg = "Jobs started in gate for 1,1." self.assertTrue(self.smtp_messages[0]['body'].startswith(start_msg)) @simple_layout('layouts/unmanaged-project.yaml') def test_unmanaged_project_start_message(self): "Test start reporting is not done for unmanaged projects." self.init_repo("org/project", tag='init') A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) self.waitUntilSettled() self.assertEqual(0, len(A.messages)) @simple_layout('layouts/merge-conflict.yaml') def test_merge_conflict_reporters(self): """Check that the config is set up correctly""" tenant = self.scheds.first.sched.abide.tenants.get('tenant-one') self.assertEqual( "Merge Failed.\n\nThis change or one of its cross-repo " "dependencies was unable to be automatically merged with the " "current state of its repository. Please rebase the change and " "upload a new patchset.", tenant.layout.pipelines['check'].merge_conflict_message) self.assertEqual( "The merge failed! For more information...", tenant.layout.pipelines['gate'].merge_conflict_message) self.assertEqual( len(tenant.layout.pipelines['check'].merge_conflict_actions), 1) self.assertEqual( len(tenant.layout.pipelines['gate'].merge_conflict_actions), 2) self.assertTrue(isinstance( tenant.layout.pipelines['check'].merge_conflict_actions[0], gerritreporter.GerritReporter)) self.assertTrue( ( isinstance(tenant.layout.pipelines['gate']. merge_conflict_actions[0], zuul.driver.smtp.smtpreporter.SMTPReporter) and isinstance(tenant.layout.pipelines['gate']. merge_conflict_actions[1], gerritreporter.GerritReporter) ) or ( isinstance(tenant.layout.pipelines['gate']. merge_conflict_actions[0], gerritreporter.GerritReporter) and isinstance(tenant.layout.pipelines['gate']. merge_conflict_actions[1], zuul.driver.smtp.smtpreporter.SMTPReporter) ) ) @simple_layout('layouts/merge-failure.yaml') def test_merge_failure_reporters(self): """Check that the config is set up correctly""" # TODO: Remove this backwards compat test in v6.0 tenant = self.scheds.first.sched.abide.tenants.get('tenant-one') self.assertEqual( "Merge Failed.\n\nThis change or one of its cross-repo " "dependencies was unable to be automatically merged with the " "current state of its repository. Please rebase the change and " "upload a new patchset.", tenant.layout.pipelines['check'].merge_conflict_message) self.assertEqual( "The merge failed! For more information...", tenant.layout.pipelines['gate'].merge_conflict_message) self.assertEqual( len(tenant.layout.pipelines['check'].merge_conflict_actions), 1) self.assertEqual( len(tenant.layout.pipelines['gate'].merge_conflict_actions), 2) self.assertTrue(isinstance( tenant.layout.pipelines['check'].merge_conflict_actions[0], gerritreporter.GerritReporter)) self.assertTrue( ( isinstance(tenant.layout.pipelines['gate']. merge_conflict_actions[0], zuul.driver.smtp.smtpreporter.SMTPReporter) and isinstance(tenant.layout.pipelines['gate']. merge_conflict_actions[1], gerritreporter.GerritReporter) ) or ( isinstance(tenant.layout.pipelines['gate']. merge_conflict_actions[0], gerritreporter.GerritReporter) and isinstance(tenant.layout.pipelines['gate']. merge_conflict_actions[1], zuul.driver.smtp.smtpreporter.SMTPReporter) ) ) def test_merge_failure_reports(self): """Check that when a change fails to merge the correct message is sent to the correct reporter""" self.commitConfigUpdate('common-config', 'layouts/merge-conflict.yaml') self.scheds.execute(lambda app: app.sched.reconfigure(app.config)) # Check a test failure isn't reported to SMTP A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') A.addApproval('Code-Review', 2) self.executor_server.failJob('project-test1', A) self.fake_gerrit.addEvent(A.addApproval('Approved', 1)) self.waitUntilSettled() self.assertEqual(3, len(self.history)) # 3 jobs self.assertEqual(0, len(self.smtp_messages)) # Check a merge failure is reported to SMTP # B should be merged, but C will conflict with B B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B') B.addPatchset({'conflict': 'foo'}) C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C') C.addPatchset({'conflict': 'bar'}) B.addApproval('Code-Review', 2) C.addApproval('Code-Review', 2) self.fake_gerrit.addEvent(B.addApproval('Approved', 1)) self.fake_gerrit.addEvent(C.addApproval('Approved', 1)) self.waitUntilSettled() self.assertEqual(6, len(self.history)) # A and B jobs self.assertEqual(1, len(self.smtp_messages)) self.assertIn('The merge failed! For more information...', self.smtp_messages[0]['body']) self.assertIn('Error merging gerrit/org/project', self.smtp_messages[0]['body']) def test_default_merge_failure_reports(self): """Check that the default merge failure reports are correct.""" # A should report success, B should report merge failure. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') A.addPatchset({'conflict': 'foo'}) B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B') B.addPatchset({'conflict': 'bar'}) A.addApproval('Code-Review', 2) B.addApproval('Code-Review', 2) self.fake_gerrit.addEvent(A.addApproval('Approved', 1)) self.fake_gerrit.addEvent(B.addApproval('Approved', 1)) self.waitUntilSettled() self.assertEqual(3, len(self.history)) # A jobs self.assertEqual(A.reported, 2) self.assertEqual(B.reported, 1) self.assertEqual(A.data['status'], 'MERGED') self.assertEqual(B.data['status'], 'NEW') self.assertIn('Build succeeded', A.messages[1]) self.assertIn('Merge Failed', B.messages[0]) self.assertIn('automatically merged', B.messages[0]) self.assertIn('Error merging gerrit/org/project', B.messages[0]) self.assertNotIn('logs.example.com', B.messages[0]) self.assertNotIn('SKIPPED', B.messages[0]) buildsets = list( self.scheds.first.connections.connections[ 'database'].getBuildsets()) self.assertEqual(buildsets[0].result, 'MERGE_CONFLICT') self.assertIn('This change or one of', buildsets[0].message) def test_submit_failure(self): A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A') A.fail_merge = True A.addApproval('Code-Review', 2) self.fake_gerrit.addEvent(A.addApproval('Approved', 1)) self.waitUntilSettled() buildsets = list( self.scheds.first.connections.connections[ 'database'].getBuildsets()) self.assertEqual(buildsets[0].result, 'MERGE_FAILURE') @simple_layout('layouts/timer-freeze-job-failure.yaml') def test_periodic_freeze_job_failure(self): self.waitUntilSettled() for x in iterate_timeout(30, 'buildset complete'): buildsets = list( self.scheds.first.connections.connections[ 'database'].getBuildsets()) if buildsets: break # Stop queuing timer triggered jobs so that the assertions # below don't race against more jobs being queued. self.commitConfigUpdate('org/common-config', 'layouts/no-timer.yaml') self.scheds.execute(lambda app: app.sched.reconfigure(app.config)) self.waitUntilSettled() # If APScheduler is in mid-event when we remove the job, we # can end up with one more event firing, so give it an extra # second to settle. time.sleep(3) self.waitUntilSettled() self.assertEqual(buildsets[0].result, 'CONFIG_ERROR') self.assertIn('Job project-test2 depends on project-test1 ' 'which was not run', buildsets[0].message) @simple_layout('layouts/freeze-job-failure.yaml') def test_freeze_job_failure(self): A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) self.waitUntilSettled() buildsets = list( self.scheds.first.connections.connections[ 'database'].getBuildsets()) self.assertEqual(buildsets[0].result, 'CONFIG_ERROR') self.assertIn('Job project-test2 depends on project-test1 ' 'which was not run', buildsets[0].message) @simple_layout('layouts/nonvoting-pipeline.yaml') def test_nonvoting_pipeline(self): "Test that a nonvoting pipeline (experimental) can still report" A = self.fake_gerrit.addFakeChange('org/experimental-project', 'master', 'A') self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) self.waitUntilSettled() self.assertEqual(self.getJobFromHistory('project-merge').result, 'SUCCESS') self.assertEqual( self.getJobFromHistory('experimental-project-test').result, 'SUCCESS') self.assertEqual(A.reported, 1) @simple_layout('layouts/disable_at.yaml') def test_disable_at(self): "Test a pipeline will only report to the disabled trigger when failing" tenant = self.scheds.first.sched.abide.tenants.get('tenant-one') self.assertEqual(3, tenant.layout.pipelines['check'].disable_at) self.assertEqual( 0, tenant.layout.pipelines['check'].state.consecutive_failures) self.assertFalse(tenant.layout.pipelines['check'].state.disabled) A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B') C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C') D = self.fake_gerrit.addFakeChange('org/project', 'master', 'D') E = self.fake_gerrit.addFakeChange('org/project', 'master', 'E') F = self.fake_gerrit.addFakeChange('org/project', 'master', 'F') G = self.fake_gerrit.addFakeChange('org/project', 'master', 'G') H = self.fake_gerrit.addFakeChange('org/project', 'master', 'H') I = self.fake_gerrit.addFakeChange('org/project', 'master', 'I') J = self.fake_gerrit.addFakeChange('org/project', 'master', 'J') K = self.fake_gerrit.addFakeChange('org/project', 'master', 'K') self.executor_server.failJob('project-test1', A) self.executor_server.failJob('project-test1', B) # Let C pass, resetting the counter self.executor_server.failJob('project-test1', D) self.executor_server.failJob('project-test1', E) self.executor_server.failJob('project-test1', F) self.executor_server.failJob('project-test1', G) self.executor_server.failJob('project-test1', H) # I also passes but should only report to the disabled reporters self.executor_server.failJob('project-test1', J) self.executor_server.failJob('project-test1', K) self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1)) self.waitUntilSettled() self.assertEqual( 2, tenant.layout.pipelines['check'].state.consecutive_failures) self.assertFalse(tenant.layout.pipelines['check'].state.disabled) self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1)) self.waitUntilSettled() self.assertEqual( 0, tenant.layout.pipelines['check'].state.consecutive_failures) self.assertFalse(tenant.layout.pipelines['check'].state.disabled) self.fake_gerrit.addEvent(D.getPatchsetCreatedEvent(1)) self.fake_gerrit.addEvent(E.getPatchsetCreatedEvent(1)) self.fake_gerrit.addEvent(F.getPatchsetCreatedEvent(1)) self.waitUntilSettled() # We should be disabled now self.assertEqual( 3, tenant.layout.pipelines['check'].state.consecutive_failures) self.assertTrue(tenant.layout.pipelines['check'].state.disabled) # We need to wait between each of these patches to make sure the # smtp messages come back in an expected order self.fake_gerrit.addEvent(G.getPatchsetCreatedEvent(1)) self.waitUntilSettled() self.fake_gerrit.addEvent(H.getPatchsetCreatedEvent(1)) self.waitUntilSettled() self.fake_gerrit.addEvent(I.getPatchsetCreatedEvent(1)) self.waitUntilSettled() # The first 6 (ABCDEF) jobs should have reported back to gerrt thus # leaving a message on each change self.assertEqual(1, len(A.messages)) self.assertIn('Build failed.', A.messages[0]) self.assertEqual(1, len(B.messages)) self.assertIn('Build failed.', B.messages[0]) self.assertEqual(1, len(C.messages)) self.assertIn('Build succeeded.', C.messages[0]) self.assertEqual(1, len(D.messages)) self.assertIn('Build failed.', D.messages[0]) self.assertEqual(1, len(E.messages)) self.assertIn('Build failed.', E.messages[0]) self.assertEqual(1, len(F.messages)) self.assertIn('Build failed.', F.messages[0]) # The last 3 (GHI) would have only reported via smtp. self.assertEqual(3, len(self.smtp_messages)) self.assertEqual(0, len(G.messages)) self.assertIn('Build failed.', self.smtp_messages[0]['body']) self.assertIn( 'project-test1 https://', self.smtp_messages[0]['body']) self.assertEqual(0, len(H.messages)) self.assertIn('Build failed.', self.smtp_messages[1]['body']) self.assertIn( 'project-test1 https://', self.smtp_messages[1]['body']) self.assertEqual(0, len(I.messages)) self.assertIn('Build succeeded.', self.smtp_messages[2]['body']) self.assertIn( 'project-test1 https://', self.smtp_messages[2]['body']) # Now reload the configuration (simulate a HUP) to check the pipeline # comes out of disabled self.scheds.execute(lambda app: app.sched.reconfigure(app.config)) self.waitUntilSettled() tenant = self.scheds.first.sched.abide.tenants.get('tenant-one') self.assertEqual(3, tenant.layout.pipelines['check'].disable_at) self.assertEqual( 0, tenant.layout.pipelines['check'].state.consecutive_failures) self.assertFalse(tenant.layout.pipelines['check'].state.disabled) self.fake_gerrit.addEvent(J.getPatchsetCreatedEvent(1)) self.fake_gerrit.addEvent(K.getPatchsetCreatedEvent(1)) self.waitUntilSettled() self.assertEqual( 2, tenant.layout.pipelines['check'].state.consecutive_failures) self.assertFalse(tenant.layout.pipelines['check'].state.disabled) # J and K went back to gerrit self.assertEqual(1, len(J.messages)) self.assertIn('Build failed.', J.messages[0]) self.assertEqual(1, len(K.messages)) self.assertIn('Build failed.', K.messages[0]) # No more messages reported via smtp self.assertEqual(3, len(self.smtp_messages)) @simple_layout('layouts/one-job-project.yaml') def test_one_job_project(self): "Test that queueing works with one job" A = self.fake_gerrit.addFakeChange('org/one-job-project', 'master', 'A') B = self.fake_gerrit.addFakeChange('org/one-job-project', 'master', 'B') A.addApproval('Code-Review', 2) B.addApproval('Code-Review', 2) self.fake_gerrit.addEvent(A.addApproval('Approved', 1)) self.fake_gerrit.addEvent(B.addApproval('Approved', 1)) self.waitUntilSettled() self.assertEqual(A.data['status'], 'MERGED') self.assertEqual(A.reported, 2) self.assertEqual(B.data['status'], 'MERGED') self.assertEqual(B.reported, 2) def test_job_aborted(self): "Test that if a execute server aborts a job, it is run again" self.executor_server.hold_jobs_in_build = True A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) self.waitUntilSettled() self.executor_server.release('.*-merge') self.waitUntilSettled() self.assertEqual(len(self.builds), 2) # first abort self.builds[0].aborted = True self.executor_server.release('.*-test*') self.waitUntilSettled() self.assertEqual(len(self.builds), 1) # second abort self.builds[0].aborted = True self.executor_server.release('.*-test*') self.waitUntilSettled() self.assertEqual(len(self.builds), 1) # third abort self.builds[0].aborted = True self.executor_server.release('.*-test*') self.waitUntilSettled() self.assertEqual(len(self.builds), 1) # fourth abort self.builds[0].aborted = True self.executor_server.release('.*-test*') self.waitUntilSettled() self.assertEqual(len(self.builds), 1) self.executor_server.hold_jobs_in_build = False self.executor_server.release() self.waitUntilSettled() self.assertEqual(len(self.history), 7) self.assertEqual(self.countJobResults(self.history, 'ABORTED'), 4) self.assertEqual(self.countJobResults(self.history, 'SUCCESS'), 3) def test_rerun_on_abort(self): "Test that if a execute server fails to run a job, it is run again" self.executor_server.hold_jobs_in_build = True A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) self.waitUntilSettled() self.executor_server.release('.*-merge') self.waitUntilSettled() self.assertEqual(len(self.builds), 2) self.builds[0].requeue = True self.executor_server.release('.*-test*') self.waitUntilSettled() tenant = self.scheds.first.sched.abide.tenants.get('tenant-one') items = tenant.layout.pipelines['check'].getAllItems() build_set = items[0].current_build_set job = list(filter(lambda j: j.name == 'project-test1', items[0].getJobs()))[0] for x in range(3): # We should have x+1 retried builds for project-test1 retry_builds = build_set.getRetryBuildsForJob(job) self.assertEqual(len(retry_builds), x + 1) for build in retry_builds: self.assertEqual(build.retry, True) self.assertEqual(build.result, 'RETRY') self.assertEqual(len(self.builds), 1, 'len of builds at x=%d is wrong' % x) self.builds[0].requeue = True self.executor_server.release('.*-test1') self.waitUntilSettled() self.executor_server.hold_jobs_in_build = False self.executor_server.release() self.waitUntilSettled() self.assertEqual(len(self.history), 6) self.assertEqual(self.countJobResults(self.history, 'SUCCESS'), 2) self.assertEqual(A.reported, 1) self.assertIn('RETRY_LIMIT', A.messages[0]) def test_executor_disconnect(self): "Test that jobs are completed after an executor disconnect" self.executor_server.hold_jobs_in_build = True A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') A.addApproval('Code-Review', 2) self.fake_gerrit.addEvent(A.addApproval('Approved', 1)) self.waitUntilSettled() # Forcibly disconnect the executor from ZK self.executor_server.zk_client.client.stop() self.executor_server.zk_client.client.start() # Find the build in the scheduler so we can check its status tenant = self.scheds.first.sched.abide.tenants.get('tenant-one') items = tenant.layout.pipelines['gate'].getAllItems() builds = items[0].current_build_set.getBuilds() build = builds[0] # Clean up the build self.scheds.first.sched.executor.cleanupLostBuildRequests() # Wait for the build to be reported as lost for x in iterate_timeout(30, 'retry build'): if build.result == 'RETRY': break # If we didn't timeout, then it worked; we're done self.executor_server.hold_jobs_in_build = False self.executor_server.release() self.waitUntilSettled() # There is a test-only race in the recording executor class # where we may record a successful first build, even though # the executor didn't actually send a build complete event. # This could probabyl be improved, but for now, it's # sufficient to verify that the job was retried. So we omit a # result classifier on the first build. self.assertHistory([ dict(name='project-merge', changes='1,1'), dict(name='project-merge', result='SUCCESS', changes='1,1'), dict(name='project-test1', result='SUCCESS', changes='1,1'), dict(name='project-test2', result='SUCCESS', changes='1,1'), ], ordered=False) # TODO: There seems to be a race condition in the kazoo election # recipe that can cause the stats election thread to hang after # reconnecting. @skip("This is unstable in the gate") def test_scheduler_disconnect(self): "Test that jobs are completed after a scheduler disconnect" self.executor_server.hold_jobs_in_build = True A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') A.addApproval('Code-Review', 2) self.fake_gerrit.addEvent(A.addApproval('Approved', 1)) self.waitUntilSettled() # Forcibly disconnect the scheduler from ZK self.scheds.execute(lambda app: app.sched.zk_client.client.stop()) self.scheds.execute(lambda app: app.sched.zk_client.client.start()) # Clean up lost builds self.scheds.first.sched.executor.cleanupLostBuildRequests() self.executor_server.hold_jobs_in_build = False self.executor_server.release() self.waitUntilSettled() self.assertHistory([ dict(name='project-merge', result='SUCCESS', changes='1,1'), dict(name='project-test1', result='SUCCESS', changes='1,1'), dict(name='project-test2', result='SUCCESS', changes='1,1'), ], ordered=False) # TODO: See comment for test_scheduler_disconnect. @skip("This is unstable in the gate") def test_zookeeper_disconnect(self): "Test that jobs are executed after a zookeeper disconnect" self.fake_nodepool.pause() A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') A.addApproval('Code-Review', 2) self.fake_gerrit.addEvent(A.addApproval('Approved', 1)) self.waitUntilSettled() self.scheds.execute(lambda app: app.sched.zk_client.client.stop()) self.scheds.execute(lambda app: app.sched.zk_client.client.start()) self.fake_nodepool.unpause() # Wait until we win the nodepool election in order to avoid a # race in waitUntilSettled with the request being fulfilled # without submitting an event. for x in iterate_timeout(60, 'nodepool election won'): found = [app for app in self.scheds if (app.sched.nodepool.election_won and app.sched.nodepool.election.is_still_valid())] if found: break self.waitUntilSettled() self.assertEqual(A.data['status'], 'MERGED') self.assertEqual(A.reported, 2) def test_nodepool_cleanup(self): "Test that we cleanup leaked node requests" self.fake_nodepool.pause() system_id = self.scheds.first.sched.system.system_id zk_nodepool = self.scheds.first.sched.nodepool.zk_nodepool req1 = zuul.model.NodeRequest(system_id, "uuid1", "tenant", "pipeline", "job", ['label'], None, 0, None) zk_nodepool.submitNodeRequest(req1, 100) req2 = zuul.model.NodeRequest("someone else", "uuid1", "tenant", "pipeline", "job", ['label'], None, 0, None) zk_nodepool.submitNodeRequest(req2, 100) self.assertEqual(zk_nodepool.getNodeRequests(), ['100-0000000000', '100-0000000001']) self.scheds.first.sched._runNodeRequestCleanup() self.assertEqual(zk_nodepool.getNodeRequests(), ['100-0000000001']) zk_nodepool.deleteNodeRequest(req2.id) def test_nodepool_failure(self): "Test that jobs are reported after a nodepool failure" self.fake_nodepool.pause() A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') A.addApproval('Code-Review', 2) self.fake_gerrit.addEvent(A.addApproval('Approved', 1)) self.waitUntilSettled() req = self.fake_nodepool.getNodeRequests()[0] self.fake_nodepool.addFailRequest(req) self.fake_nodepool.unpause() self.waitUntilSettled() self.assertEqual(A.data['status'], 'NEW') self.assertEqual(A.reported, 2) self.assertTrue(re.search('project-merge .* NODE_FAILURE', A.messages[1])) self.assertTrue( 'Skipped due to failed job project-merge' in A.messages[1]) self.assertReportedStat( 'zuul.nodepool.requests.requested.total', value='1', kind='c') self.assertReportedStat( 'zuul.nodepool.requests.requested.label.label1', value='1', kind='c') self.assertReportedStat( 'zuul.nodepool.requests.failed.label.label1', value='1', kind='c') self.assertReportedStat( 'zuul.tenant.tenant-one.pipeline.gate.project.review_example_com.' 'org_project.master.job.project-merge.NODE_FAILURE', value='1', kind='c') def test_nodepool_resources(self): "Test that resources are reported" self.executor_server.hold_jobs_in_build = True self.fake_nodepool.resources = { 'cores': 2, 'ram': 1024, 'instances': 1, } A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') A.addApproval('Code-Review', 2) self.fake_gerrit.addEvent(A.addApproval('Approved', 1)) self.waitUntilSettled() self.executor_server.release('project-merge') self.waitUntilSettled() self.waitUntilNodeCacheSync( self.scheds.first.sched.nodepool.zk_nodepool) self.scheds.first.sched._runStats() # Check that resource usage gauges are reported self.assertHistory([ dict(name='project-merge', result='SUCCESS', changes='1,1'), ]) self.assertReportedStat( 'zuul.nodepool.resources.total.tenant.tenant-one.cores', value='6', kind='g') self.assertReportedStat( 'zuul.nodepool.resources.total.tenant.tenant-one.ram', value='3072', kind='g') self.assertReportedStat( 'zuul.nodepool.resources.total.tenant.tenant-one.instances', value='3', kind='g') # All 3 nodes are in use self.assertReportedStat( 'zuul.nodepool.resources.in_use.tenant.tenant-one.cores', value='6', kind='g') self.assertReportedStat( 'zuul.nodepool.resources.in_use.tenant.tenant-one.ram', value='3072', kind='g') self.assertReportedStat( 'zuul.nodepool.resources.in_use.tenant.tenant-one.instances', value='3', kind='g') self.assertReportedStat( 'zuul.nodepool.resources.in_use.project.review_example_com/org/' 'project.cores', value='6', kind='g') self.assertReportedStat( 'zuul.nodepool.resources.in_use.project.review_example_com/org/' 'project.ram', value='3072', kind='g') self.assertReportedStat( 'zuul.nodepool.resources.in_use.project.review_example_com/org/' 'project.instances', value='3', kind='g') # Check that resource usage counters are reported self.assertReportedStat( 'zuul.nodepool.resources.in_use.tenant.tenant-one.cores', kind='c') self.assertReportedStat( 'zuul.nodepool.resources.in_use.tenant.tenant-one.ram', kind='c') self.assertReportedStat( 'zuul.nodepool.resources.in_use.tenant.tenant-one.instances', kind='c') self.assertReportedStat( 'zuul.nodepool.resources.in_use.project.review_example_com/org/' 'project.cores', kind='c') self.assertReportedStat( 'zuul.nodepool.resources.in_use.project.review_example_com/org/' 'project.ram', kind='c') self.assertReportedStat( 'zuul.nodepool.resources.in_use.project.review_example_com/org/' 'project.instances', kind='c') self.executor_server.hold_jobs_in_build = False self.executor_server.release() self.waitUntilSettled() self.assertEqual(A.data['status'], 'MERGED') self.assertEqual(A.reported, 2) def test_nodepool_pipeline_priority(self): "Test that nodes are requested at the correct pipeline priority" self.fake_nodepool.pause() A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') self.fake_gerrit.addEvent(A.getRefUpdatedEvent()) self.waitUntilSettled() B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B') self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1)) self.waitUntilSettled() C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C') C.addApproval('Code-Review', 2) self.fake_gerrit.addEvent(C.addApproval('Approved', 1)) self.waitUntilSettled() reqs = self.fake_nodepool.getNodeRequests() # The requests come back sorted by priority. Since we have # three requests for the three changes each with a different # priority. Also they get a serial number based on order they # were received so the number on the endof the oid should map # to order submitted. # * gate first - high priority - change C self.assertEqual(reqs[0]['_oid'], '100-0000000002') self.assertEqual(reqs[0]['node_types'], ['label1']) # * check second - normal priority - change B self.assertEqual(reqs[1]['_oid'], '200-0000000001') self.assertEqual(reqs[1]['node_types'], ['label1']) # * post third - low priority - change A # additionally, the post job defined uses an ubuntu-xenial node, # so we include that check just as an extra verification self.assertEqual(reqs[2]['_oid'], '300-0000000000') self.assertEqual(reqs[2]['node_types'], ['ubuntu-xenial']) self.fake_nodepool.unpause() self.waitUntilSettled() @simple_layout('layouts/two-projects-integrated.yaml') def test_nodepool_relative_priority_check(self): "Test that nodes are requested at the relative priority" self.fake_nodepool.pause() A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) self.waitUntilSettled() B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B') self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1)) self.waitUntilSettled() C = self.fake_gerrit.addFakeChange('org/project1', 'master', 'C') self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1)) self.waitUntilSettled() D = self.fake_gerrit.addFakeChange('org/project2', 'master', 'D') self.fake_gerrit.addEvent(D.getPatchsetCreatedEvent(1)) self.waitUntilSettled() reqs = self.fake_nodepool.getNodeRequests() # The requests come back sorted by priority. # Change A, first change for project, high relative priority. self.assertEqual(reqs[0]['_oid'], '200-0000000000') self.assertEqual(reqs[0]['relative_priority'], 0) # Change C, first change for project1, high relative priority. self.assertEqual(reqs[1]['_oid'], '200-0000000002') self.assertEqual(reqs[1]['relative_priority'], 0) # Change B, second change for project, lower relative priority. self.assertEqual(reqs[2]['_oid'], '200-0000000001') self.assertEqual(reqs[2]['relative_priority'], 1) # Change D, first change for project2 shared with project1, # lower relative priority than project1. self.assertEqual(reqs[3]['_oid'], '200-0000000003') self.assertEqual(reqs[3]['relative_priority'], 1) # Fulfill only the first request self.fake_nodepool.fulfillRequest(reqs[0]) for x in iterate_timeout(30, 'fulfill request'): reqs = list(self.scheds.first.sched.nodepool.getNodeRequests()) if len(reqs) < 4: break self.waitUntilSettled() reqs = self.fake_nodepool.getNodeRequests() # Change B, now first change for project, equal priority. self.assertEqual(reqs[0]['_oid'], '200-0000000001') self.assertEqual(reqs[0]['relative_priority'], 0) # Change C, now first change for project1, equal priority. self.assertEqual(reqs[1]['_oid'], '200-0000000002') self.assertEqual(reqs[1]['relative_priority'], 0) # Change D, first change for project2 shared with project1, # still lower relative priority than project1. self.assertEqual(reqs[2]['_oid'], '200-0000000003') self.assertEqual(reqs[2]['relative_priority'], 1) self.fake_nodepool.unpause() self.waitUntilSettled() @simple_layout('layouts/two-projects-integrated.yaml') def test_nodepool_relative_priority_long(self): "Test that nodes are requested at the relative priority" self.fake_nodepool.pause() count = 13 changes = [] for x in range(count): change = self.fake_gerrit.addFakeChange( 'org/project', 'master', 'A') self.fake_gerrit.addEvent(change.getPatchsetCreatedEvent(1)) self.waitUntilSettled() changes.append(change) reqs = self.fake_nodepool.getNodeRequests() self.assertEqual(len(reqs), 13) # The requests come back sorted by priority. for x in range(10): self.assertEqual(reqs[x]['relative_priority'], x) self.assertEqual(reqs[10]['relative_priority'], 10) self.assertEqual(reqs[11]['relative_priority'], 10) self.assertEqual(reqs[12]['relative_priority'], 10) # Fulfill only the first request self.fake_nodepool.fulfillRequest(reqs[0]) for x in iterate_timeout(30, 'fulfill request'): reqs = list(self.scheds.first.sched.nodepool.getNodeRequests()) if len(reqs) < count: break self.waitUntilSettled() reqs = self.fake_nodepool.getNodeRequests() self.assertEqual(len(reqs), 12) for x in range(10): self.assertEqual(reqs[x]['relative_priority'], x) self.assertEqual(reqs[10]['relative_priority'], 10) self.assertEqual(reqs[11]['relative_priority'], 10) self.fake_nodepool.unpause() self.waitUntilSettled() @simple_layout('layouts/two-projects-integrated.yaml') def test_nodepool_relative_priority_gate(self): "Test that nodes are requested at the relative priority" self.fake_nodepool.pause() A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A') A.addApproval('Code-Review', 2) self.fake_gerrit.addEvent(A.addApproval('Approved', 1)) self.waitUntilSettled() B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B') B.addApproval('Code-Review', 2) self.fake_gerrit.addEvent(B.addApproval('Approved', 1)) self.waitUntilSettled() # project does not share a queue with project1 and project2. C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C') C.addApproval('Code-Review', 2) self.fake_gerrit.addEvent(C.addApproval('Approved', 1)) self.waitUntilSettled() reqs = self.fake_nodepool.getNodeRequests() # The requests come back sorted by priority. # Change A, first change for shared queue, high relative # priority. self.assertEqual(reqs[0]['_oid'], '100-0000000000') self.assertEqual(reqs[0]['relative_priority'], 0) # Change C, first change for independent project, high # relative priority. self.assertEqual(reqs[1]['_oid'], '100-0000000002') self.assertEqual(reqs[1]['relative_priority'], 0) # Change B, second change for shared queue, lower relative # priority. self.assertEqual(reqs[2]['_oid'], '100-0000000001') self.assertEqual(reqs[2]['relative_priority'], 1) self.fake_nodepool.unpause() self.waitUntilSettled() def test_nodepool_project_removal(self): "Test that nodes are returned unused after project removal" self.fake_nodepool.pause() A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A') self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) self.waitUntilSettled() self.newTenantConfig('config/single-tenant/main-one-project.yaml') # This layout defines only org/project, not org/project1 self.commitConfigUpdate( 'common-config', 'layouts/live-reconfiguration-del-project.yaml') self.scheds.execute(lambda app: app.sched.reconfigure(app.config)) self.waitUntilSettled() self.fake_nodepool.unpause() self.waitUntilSettled() self.assertEqual(A.data['status'], 'NEW') self.assertEqual(A.reported, 0) for node in self.fake_nodepool.getNodes(): self.assertFalse(node['_lock']) self.assertEqual(node['state'], 'ready') @simple_layout('layouts/nodeset-fallback.yaml') def test_nodeset_fallback(self): # Test that nodeset fallback works self.executor_server.hold_jobs_in_build = True # Verify that we get the correct number and order of # alternates from our nested config. tenant = self.scheds.first.sched.abide.tenants.get('tenant-one') job = tenant.layout.getJob('check-job') alts = job.flattenNodesetAlternatives(tenant.layout) self.assertEqual(4, len(alts)) self.assertEqual('fast-nodeset', alts[0].name) self.assertEqual('', alts[1].name) self.assertEqual('red-nodeset', alts[2].name) self.assertEqual('blue-nodeset', alts[3].name) self.fake_nodepool.pause() A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) self.waitUntilSettled() req = self.fake_nodepool.getNodeRequests()[0] self.fake_nodepool.addFailRequest(req) self.fake_nodepool.unpause() self.waitUntilSettled() build = self.getBuildByName('check-job') inv_path = os.path.join(build.jobdir.root, 'ansible', 'inventory.yaml') with open(inv_path, 'r') as f: inventory = yaml.safe_load(f) label = inventory['all']['hosts']['controller']['nodepool']['label'] self.assertEqual('slow-label', label) self.executor_server.hold_jobs_in_build = False self.executor_server.release() self.waitUntilSettled() self.assertEqual(A.data['status'], 'NEW') self.assertEqual(A.reported, 1) self.assertNotIn('NODE_FAILURE', A.messages[0]) self.assertHistory([ dict(name='check-job', result='SUCCESS', changes='1,1'), ], ordered=False) @simple_layout('layouts/multiple-templates.yaml') def test_multiple_project_templates(self): # Test that applying multiple project templates to a project # doesn't alter them when used for a second project. A = self.fake_gerrit.addFakeChange('org/project2', 'master', 'A') self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) self.waitUntilSettled() build = self.getJobFromHistory('py27') self.assertEqual(build.parameters['zuul']['jobtags'], []) def test_pending_merge_in_reconfig(self): # Test that if we are waiting for an outstanding merge on # reconfiguration that we continue to do so. self.hold_merge_jobs_in_queue = True A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A') A.setMerged() self.fake_gerrit.addEvent(A.getRefUpdatedEvent()) self.waitUntilSettled() jobs = list(self.merger_api.all()) self.assertEqual(len(jobs), 1) self.assertEqual(jobs[0].state, zuul.model.MergeRequest.HOLD) # Reconfigure while we still have an outstanding merge job self.hold_merge_jobs_in_queue = False tenant = self.scheds.first.sched.abide.tenants.get('tenant-one') (trusted, project1) = tenant.getProject('org/project1') event = zuul.model.TriggerEvent() self.scheds.first.sched.reconfigureTenant( self.scheds.first.sched.abide.tenants['tenant-one'], project1, event) self.waitUntilSettled() # Verify the merge job is still running and that the item is # in the pipeline jobs = list(self.merger_api.all()) self.assertEqual(jobs[0].state, zuul.model.MergeRequest.HOLD) self.assertEqual(len(jobs), 1) pipeline = tenant.layout.pipelines['post'] self.assertEqual(len(pipeline.getAllItems()), 1) self.merger_api.release() self.waitUntilSettled() jobs = list(self.merger_api.all()) self.assertEqual(len(jobs), 0) @simple_layout('layouts/parent-matchers.yaml') def test_parent_matchers(self): "Test that if a job's parent does not match, the job does not run" A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) self.waitUntilSettled() self.assertHistory([]) files = {'foo.txt': ''} B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B', files=files) self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1)) self.waitUntilSettled() files = {'bar.txt': ''} C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C', files=files) self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1)) self.waitUntilSettled() files = {'foo.txt': '', 'bar.txt': ''} D = self.fake_gerrit.addFakeChange('org/project', 'master', 'D', files=files) self.fake_gerrit.addEvent(D.getPatchsetCreatedEvent(1)) self.waitUntilSettled() self.assertHistory([ dict(name='child-job', result='SUCCESS', changes='3,1'), dict(name='child-job', result='SUCCESS', changes='4,1'), ], ordered=False) @simple_layout('layouts/file-matchers.yaml') def test_file_matchers(self): "Test several file matchers" files = {'parent1.txt': ''} A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A', files=files) self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) self.waitUntilSettled() files = {'parent2.txt': ''} B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B', files=files) self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1)) self.waitUntilSettled() files = {'child.txt': ''} C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C', files=files) self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1)) self.waitUntilSettled() files = {'project.txt': ''} D = self.fake_gerrit.addFakeChange('org/project', 'master', 'D', files=files) self.fake_gerrit.addEvent(D.getPatchsetCreatedEvent(1)) self.waitUntilSettled() files = {'tests/foo': ''} E = self.fake_gerrit.addFakeChange('org/project', 'master', 'E', files=files) self.fake_gerrit.addEvent(E.getPatchsetCreatedEvent(1)) self.waitUntilSettled() files = {'tests/docs/foo': ''} F = self.fake_gerrit.addFakeChange('org/project', 'master', 'F', files=files) self.fake_gerrit.addEvent(F.getPatchsetCreatedEvent(1)) self.waitUntilSettled() self.assertHistory([ dict(name='child-job', result='SUCCESS', changes='2,1'), dict(name='child-override-job', result='SUCCESS', changes='3,1'), dict(name='project-override-job', result='SUCCESS', changes='4,1'), dict(name='irr-job', result='SUCCESS', changes='5,1'), dict(name='irr-override-job', result='SUCCESS', changes='5,1'), dict(name='irr-job', result='SUCCESS', changes='6,1'), ], ordered=False) def test_trusted_project_dep_on_non_live_untrusted_project(self): # Test we get a layout for trusted projects when they depend on # non live untrusted projects. This checks against a bug where # trusted project config changes can end up in a infinite loop # trying to find the right layout. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') files = {'zuul.yaml': ''} B = self.fake_gerrit.addFakeChange('common-config', 'master', 'B', files=files) B.setDependsOn(A, 1) self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1)) self.waitUntilSettled() self.assertHistory([ dict(name='project-merge', result='SUCCESS', changes='1,1 2,1'), dict(name='project-test1', result='SUCCESS', changes='1,1 2,1'), dict(name='project-test2', result='SUCCESS', changes='1,1 2,1'), dict(name='project1-project2-integration', result='SUCCESS', changes='1,1 2,1'), ], ordered=False) @simple_layout('layouts/success-message.yaml') def test_success_message(self): # Test the success_message (and failure_message) job attrs A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A') self.executor_server.failJob('badjob', A) self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) self.waitUntilSettled() self.assertEqual(len(A.messages), 1) self.assertTrue('YAY' in A.messages[0]) self.assertTrue('BOO' in A.messages[0]) def test_merge_error(self): # Test we don't get stuck on a merger error self.waitUntilSettled() self.patch(zuul.merger.merger.Repo, 'retry_attempts', 1) A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') self.patch(git.Git, 'GIT_PYTHON_GIT_EXECUTABLE', os.path.join(FIXTURE_DIR, 'git_fail.sh')) self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) self.waitUntilSettled() self.assertIn('Unable to update gerrit/org/project', A.messages[0]) @simple_layout('layouts/vars.yaml') def test_jobdata(self): # Test the use of JobData objects for job variables self.executor_server.hold_jobs_in_build = True self.useFixture(fixtures.MonkeyPatch( 'zuul.model.FrozenJob.MAX_DATA_LEN', 1)) self.useFixture(fixtures.MonkeyPatch( 'zuul.model.Build.MAX_DATA_LEN', 1)) # Return some data and pause the job. We use a paused job # here because there are only two times we refresh JobData: # 1) A job which has not yet started its build # because the waiting status may change, we refresh the FrozenJob # 2) A job which is paused # because the result/data may change, we refresh the Build # This allows us to test that we re-use JobData instances when # we are able. A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') self.executor_server.returnData( 'check-job', A, {'somedata': 'foobar', 'zuul': {'pause': True}}, ) self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) self.waitUntilSettled() tenant = self.scheds.first.sched.abide.tenants.get('tenant-one') item = tenant.layout.pipelines['check'].queues[0].queue[0] hold_job = item.getJobs()[1] # Refresh the pipeline so that we can verify the JobData # objects are immutable. old_hold_job_variables = hold_job.variables ctx = self.refreshPipelines(self.scheds.first.sched) new_hold_job_variables = hold_job.variables self.executor_server.release('check-job') self.waitUntilSettled() # Waiting on hold-job now # Check the assertions on the hold job here so that the test # can fail normally if they fail (it times out otherwise due # to the held job). self.assertEqual('hold-job', hold_job.name) # Make sure we're really using JobData objects self.assertTrue(isinstance(hold_job._variables, zuul.model.JobData)) # Make sure the same object instance is used self.assertIs(old_hold_job_variables, new_hold_job_variables) # Hopefully these asserts won't change much over time. If # they don't they may be a good way for us to catch unintended # extra read operations. If they change too much, they may # not be worth keeping and we can just remove them. self.assertEqual(5, ctx.cumulative_read_objects) self.assertEqual(5, ctx.cumulative_read_znodes) self.assertEqual(0, ctx.cumulative_write_objects) self.assertEqual(0, ctx.cumulative_write_znodes) check_job = item.getJobs()[0] self.assertEqual('check-job', check_job.name) self.assertTrue(isinstance(check_job._variables, zuul.model.JobData)) check_build = item.current_build_set.getBuild(check_job) self.assertTrue(isinstance(check_build._result_data, zuul.model.JobData)) # Refresh the pipeline so that we can verify the JobData # objects are immutable. old_check_build_results = check_build.result_data ctx = self.refreshPipelines(self.scheds.first.sched) new_check_build_results = check_build.result_data # Verify that we did not reload results self.assertIs(old_check_build_results, new_check_build_results) # Again check the object read counts self.assertEqual(4, ctx.cumulative_read_objects) self.assertEqual(4, ctx.cumulative_read_znodes) self.assertEqual(0, ctx.cumulative_write_objects) self.assertEqual(0, ctx.cumulative_write_znodes) self.executor_server.hold_jobs_in_build = False self.executor_server.release() self.waitUntilSettled() self.assertHistory([ dict(name='check-job', result='SUCCESS', changes='1,1'), dict(name='hold-job', result='SUCCESS', changes='1,1'), ], ordered=False) def test_zkobject_parallel_refresh(self): # Test that we don't deadlock when refreshing objects zkobject.BaseZKContext._max_workers = 1 self.executor_server.hold_jobs_in_build = True A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B') A.addApproval('Code-Review', 2) B.addApproval('Code-Review', 2) self.fake_gerrit.addEvent(A.addApproval('Approved', 1)) self.fake_gerrit.addEvent(B.addApproval('Approved', 1)) self.waitUntilSettled() self.executor_server.hold_jobs_in_build = False self.executor_server.release() self.waitUntilSettled() self.assertHistory([ dict(name='project-merge', result='SUCCESS', changes='1,1'), dict(name='project-test1', result='SUCCESS', changes='1,1'), dict(name='project-test2', result='SUCCESS', changes='1,1'), dict(name='project-merge', result='SUCCESS', changes='1,1 2,1'), dict(name='project-test1', result='SUCCESS', changes='1,1 2,1'), dict(name='project-test2', result='SUCCESS', changes='1,1 2,1'), ], ordered=False) self.assertEqual(A.data['status'], 'MERGED') self.assertEqual(B.data['status'], 'MERGED') def test_leaked_pipeline_cleanup(self): self.waitUntilSettled() sched = self.scheds.first.sched pipeline_state_path = "/zuul/tenant/tenant-one/pipeline/invalid" self.zk_client.client.ensure_path(pipeline_state_path) # Create the ZK path as a side-effect of getting the event queue. sched.pipeline_management_events["tenant-one"]["invalid"] pipeline_event_queue_path = PIPELINE_NAME_ROOT.format( tenant="tenant-one", pipeline="invalid") self.assertIsNotNone(self.zk_client.client.exists(pipeline_state_path)) # Wait for the event watcher to create the event queues for _ in iterate_timeout(30, "create event queues"): for event_queue in ("management", "trigger", "result"): if self.zk_client.client.exists( f"{pipeline_event_queue_path}/{event_queue}") is None: break else: break sched._runLeakedPipelineCleanup() self.assertIsNone( self.zk_client.client.exists(pipeline_event_queue_path)) self.assertIsNone(self.zk_client.client.exists(pipeline_state_path)) class TestChangeQueues(ZuulTestCase): tenant_config_file = 'config/change-queues/main.yaml' def _test_dependent_queues_per_branch(self, project, queue_name='integrated', queue_repo='common-config'): self.create_branch(project, 'stable') self.fake_gerrit.addEvent( self.fake_gerrit.getFakeBranchCreatedEvent(project, 'stable')) self.waitUntilSettled() self.executor_server.hold_jobs_in_build = True A = self.fake_gerrit.addFakeChange(project, 'master', 'A') B = self.fake_gerrit.addFakeChange(project, 'stable', 'B') A.addApproval('Code-Review', 2) B.addApproval('Code-Review', 2) self.executor_server.failJob('project-test', A) # Let first go A into gate then B self.fake_gerrit.addEvent(A.addApproval('Approved', 1)) self.waitUntilSettled() self.fake_gerrit.addEvent(B.addApproval('Approved', 1)) self.waitUntilSettled() # There should be one project-test job at the head of each queue self.assertBuilds([ dict(name='project-test', changes='1,1'), dict(name='project-test', changes='2,1'), ]) tenant = self.scheds.first.sched.abide.tenants.get('tenant-one') _, p = tenant.getProject(project) q1 = tenant.layout.pipelines['gate'].getQueue( p.canonical_name, 'master') q2 = tenant.layout.pipelines['gate'].getQueue( p.canonical_name, 'stable') self.assertEqual(q1.name, queue_name) self.assertEqual(q2.name, queue_name) # Both queues must contain one item self.assertEqual(len(q1.queue), 1) self.assertEqual(len(q2.queue), 1) # Fail job on the change on master self.executor_server.hold_jobs_in_build = False self.executor_server.release() self.waitUntilSettled() self.assertNotEqual(A.data['status'], 'MERGED') self.assertEqual(B.data['status'], 'MERGED') self.assertEqual(A.reported, 2) self.assertEqual(B.reported, 2) # Now reconfigure the queue to be non-branched and run the same test # again. conf = textwrap.dedent( """ - queue: name: {} per-branch: false """).format(queue_name) file_dict = {'zuul.d/queue.yaml': conf} C = self.fake_gerrit.addFakeChange(queue_repo, 'master', 'A', files=file_dict) C.setMerged() self.fake_gerrit.addEvent(C.getChangeMergedEvent()) self.waitUntilSettled() self.executor_server.hold_jobs_in_build = True D = self.fake_gerrit.addFakeChange(project, 'master', 'D') E = self.fake_gerrit.addFakeChange(project, 'stable', 'E') D.addApproval('Code-Review', 2) E.addApproval('Code-Review', 2) self.executor_server.failJob('project-test', D) # Let first go A into gate then B self.fake_gerrit.addEvent(D.addApproval('Approved', 1)) self.waitUntilSettled() self.fake_gerrit.addEvent(E.addApproval('Approved', 1)) self.waitUntilSettled() # There should be two project-test jobs in a shared queue self.assertBuilds([ dict(name='project-test', changes='4,1'), dict(name='project-test', changes='4,1 5,1'), ]) tenant = self.scheds.first.sched.abide.tenants.get('tenant-one') _, p = tenant.getProject(project) q1 = tenant.layout.pipelines['gate'].getQueue( p.canonical_name, 'master') q2 = tenant.layout.pipelines['gate'].getQueue( p.canonical_name, 'stable') q3 = tenant.layout.pipelines['gate'].getQueue( p.canonical_name, None) # There should be no branch specific queues anymore self.assertEqual(q1, None) self.assertEqual(q2, None) self.assertEqual(q3.name, queue_name) # Both queues must contain one item self.assertEqual(len(q3.queue), 2) # Release project-test of D to make history after test deterministic self.executor_server.release('project-test', change='4 1') self.waitUntilSettled() # Fail job on the change on master self.executor_server.hold_jobs_in_build = False self.executor_server.release() self.waitUntilSettled() self.assertNotEqual(D.data['status'], 'MERGED') self.assertEqual(E.data['status'], 'MERGED') self.assertEqual(D.reported, 2) self.assertEqual(E.reported, 2) self.assertHistory([ # Independent runs because of per branch queues dict(name='project-test', result='FAILURE', changes='1,1'), dict(name='project-test', result='SUCCESS', changes='2,1'), # Same queue with gate reset because of 4,1 dict(name='project-test', result='FAILURE', changes='4,1'), # Result can be anything depending on timing of the gate reset. dict(name='project-test', changes='4,1 5,1'), dict(name='project-test', result='SUCCESS', changes='5,1'), ], ordered=False) def test_dependent_queues_per_branch(self): """ Test that change queues can be different for different branches. In this case the project contains zuul config so the branches are known upfront and the queues are pre-seeded. """ self._test_dependent_queues_per_branch('org/project') def test_dependent_queues_per_branch_no_config(self): """ Test that change queues can be different for different branches. In this case we create changes for two branches in a repo that doesn't contain zuul config so the queues are not pre-seeded in the gate pipeline. """ self._test_dependent_queues_per_branch('org/project2') def test_dependent_queues_per_branch_untrusted(self): """ Test that change queues can be different for different branches. In this case we create changes for two branches in an untrusted repo that defines its own queue. """ self._test_dependent_queues_per_branch( 'org/project3', queue_name='integrated-untrusted', queue_repo='org/project3') def test_dependent_queues_per_branch_project_queue(self): """ Test that change queues can be different for different branches. In this case we create changes for two branches in a repo that references the queue on project level instead of pipeline level. """ self._test_dependent_queues_per_branch('org/project4') def test_duplicate_definition_on_branches(self): project = 'org/project3' self.create_branch(project, 'stable') self.fake_gerrit.addEvent( self.fake_gerrit.getFakeBranchCreatedEvent(project, 'stable')) self.waitUntilSettled() tenant = self.scheds.first.sched.abide.tenants.get('tenant-one') self.assertEquals( len(tenant.layout.loading_errors), 1, "No error should have been accumulated") # This error is expected and unrelated to this test (the # ignored configuration is used by other tests in this class): self.assertIn('Queue integrated already defined', tenant.layout.loading_errors[0].error) # At this point we've verified that we can have identical # queue definitions on multiple branches without conflict. # Next, let's try to change the queue def on one branch so it # doesn't match (flip the per-branch boolean): conf = textwrap.dedent( """ - queue: name: integrated-untrusted per-branch: false """) file_dict = {'zuul.d/queue.yaml': conf} A = self.fake_gerrit.addFakeChange(project, 'stable', 'A', files=file_dict) A.addApproval('Code-Review', 2) self.fake_gerrit.addEvent(A.addApproval('Approved', 1)) self.waitUntilSettled() self.assertEqual(len(A.messages), 1) self.assertTrue( 'Queue integrated-untrusted does not match ' 'existing definition in branch master' in A.messages[0]) self.assertEqual(A.data['status'], 'NEW') class TestJobUpdateBrokenConfig(ZuulTestCase): tenant_config_file = 'config/job-update-broken/main.yaml' def test_fix_check_without_running(self): "Test that we can fix a broken check pipeline (don't run the job)" in_repo_conf = textwrap.dedent( """ - job: name: existing-files files: - README.txt - project-template: name: files-template check: jobs: - existing-files - noop """) # When the config is broken, we don't override any files # matchers since we don't have a valid basis. Since this # doesn't update README.txt, nothing should run. file_dict = {'zuul.d/existing.yaml': in_repo_conf} A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A', files=file_dict) self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) self.waitUntilSettled() self.assertHistory([]) self.assertEqual(A.reported, 1) def test_fix_check_with_running(self): "Test that we can fix a broken check pipeline (do run the job)" in_repo_conf = textwrap.dedent( """ - job: name: existing-files files: - README.txt - project-template: name: files-template check: jobs: - existing-files """) # When the config is broken, we don't override any files # matchers since we don't have a valid basis. Since this # does update README.txt, the job should run. file_dict = {'zuul.d/existing.yaml': in_repo_conf, 'README.txt': ''} A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A', files=file_dict) self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) self.waitUntilSettled() self.assertHistory([ dict(name='existing-files', result='SUCCESS', changes='1,1'), ]) self.assertEqual(A.reported, 1) class TestJobUpdateFileMatcher(ZuulTestCase): tenant_config_file = 'config/job-update/main.yaml' def test_matchers(self): "Test matchers work as expected with no change" file_dict = {'README.txt': ''} A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A', files=file_dict) self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) self.waitUntilSettled() file_dict = {'something_else': ''} B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B', files=file_dict) self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1)) self.waitUntilSettled() self.assertHistory([ dict(name='existing-files', result='SUCCESS', changes='1,1'), dict(name='existing-irr', result='SUCCESS', changes='2,1'), ]) def test_job_update(self): "Test matchers are overridden with a config update" in_repo_conf = textwrap.dedent( """ - job: name: existing-files tags: foo - job: name: existing-irr tags: foo """) file_dict = {'zuul.d/new.yaml': in_repo_conf} A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A', files=file_dict) self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) self.waitUntilSettled() self.assertHistory([ dict(name='existing-files', result='SUCCESS', changes='1,1'), dict(name='existing-irr', result='SUCCESS', changes='1,1'), ], ordered=False) def test_job_update_files(self): "Test that changes to file matchers themselves don't run jobs" # Normally we want to ignore file matchers and run jobs if the # job config changes, but if the only thing about the job # config that changes *is* the file matchers, then we don't # want to run it. in_repo_conf = textwrap.dedent( """ - job: name: existing-files files: 'doesnotexist' - job: name: existing-irr irrelevant-files: - README - ^zuul.d/.*$ - newthing """) file_dict = {'zuul.d/new.yaml': in_repo_conf} A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A', files=file_dict) self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) self.waitUntilSettled() self.assertHistory([]) def test_new_job(self): "Test matchers are overridden when creating a new job" in_repo_conf = textwrap.dedent( """ - job: name: new-files parent: existing-files - project: check: jobs: - new-files """) file_dict = {'zuul.d/new.yaml': in_repo_conf} A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A', files=file_dict) self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) self.waitUntilSettled() self.assertHistory([ dict(name='new-files', result='SUCCESS', changes='1,1'), ]) def test_patch_series(self): "Test that we diff to the nearest layout in a patch series" in_repo_conf = textwrap.dedent( """ - job: name: new-files1 parent: existing-files - project: check: jobs: - new-files1 """) file_dict = {'zuul.d/new1.yaml': in_repo_conf} A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A', files=file_dict) in_repo_conf = textwrap.dedent( """ - job: name: new-files2 parent: existing-files - project: check: jobs: - new-files2 """) file_dict = {'zuul.d/new2.yaml': in_repo_conf} B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B', files=file_dict) B.setDependsOn(A, 1) self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1)) self.waitUntilSettled() self.assertHistory([ dict(name='new-files2', result='SUCCESS', changes='1,1 2,1'), ]) def test_disable_match(self): "Test matchers are not overridden if we say so" in_repo_conf = textwrap.dedent( """ - job: name: new-files parent: existing-files match-on-config-updates: false - project: check: jobs: - new-files """) file_dict = {'zuul.d/new.yaml': in_repo_conf} A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A', files=file_dict) self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) self.waitUntilSettled() self.assertHistory([]) class TestExecutor(ZuulTestCase): tenant_config_file = 'config/single-tenant/main.yaml' def assertFinalState(self): # In this test, we expect to shut down in a non-final state, # so skip these checks. pass def assertCleanShutdown(self): self.log.debug("Assert clean shutdown") # After shutdown, make sure no jobs are running self.assertEqual({}, self.executor_server.job_workers) # Make sure that git.Repo objects have been garbage collected. gc.disable() try: gc.collect() for obj in gc.get_objects(): if isinstance(obj, git.Repo): self.log.debug("Leaked git repo object: %s" % repr(obj)) gc.enable() finally: gc.enable() def test_executor_shutdown(self): "Test that the executor can shut down with jobs running" self.executor_server.hold_jobs_in_build = True A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') A.addApproval('Code-Review', 2) self.fake_gerrit.addEvent(A.addApproval('Approved', 1)) self.waitUntilSettled() class TestDependencyGraph(ZuulTestCase): tenant_config_file = 'config/dependency-graph/main.yaml' def test_dependeny_graph_dispatch_jobs_once(self): "Test a job in a dependency graph is queued only once" # Job dependencies, starting with A # A # / \ # B C # / \ / \ # D F E # | # G self.executor_server.hold_jobs_in_build = True change = self.fake_gerrit.addFakeChange( 'org/project', 'master', 'change') change.addApproval('Code-Review', 2) self.fake_gerrit.addEvent(change.addApproval('Approved', 1)) self.waitUntilSettled() self.assertEqual([b.name for b in self.builds], ['A']) self.executor_server.release('A') self.waitUntilSettled() self.assertEqual(sorted(b.name for b in self.builds), ['B', 'C']) self.executor_server.release('B') self.waitUntilSettled() self.assertEqual(sorted(b.name for b in self.builds), ['C', 'D']) self.executor_server.release('D') self.waitUntilSettled() self.assertEqual([b.name for b in self.builds], ['C']) self.executor_server.release('C') self.waitUntilSettled() self.assertEqual(sorted(b.name for b in self.builds), ['E', 'F']) self.executor_server.release('F') self.waitUntilSettled() self.assertEqual(sorted(b.name for b in self.builds), ['E', 'G']) self.executor_server.release('G') self.waitUntilSettled() self.assertEqual([b.name for b in self.builds], ['E']) self.executor_server.release('E') self.waitUntilSettled() self.assertEqual(len(self.builds), 0) self.executor_server.hold_jobs_in_build = False self.executor_server.release() self.waitUntilSettled() self.assertEqual(len(self.builds), 0) self.assertEqual(len(self.history), 7) self.assertEqual(change.data['status'], 'MERGED') self.assertEqual(change.reported, 2) def test_jobs_launched_only_if_all_dependencies_are_successful(self): "Test that a job waits till all dependencies are successful" # Job dependencies, starting with A # A # / \ # B C* # / \ / \ # D F E # | # G self.executor_server.hold_jobs_in_build = True change = self.fake_gerrit.addFakeChange( 'org/project', 'master', 'change') change.addApproval('Code-Review', 2) self.executor_server.failJob('C', change) self.fake_gerrit.addEvent(change.addApproval('Approved', 1)) self.waitUntilSettled() self.assertEqual([b.name for b in self.builds], ['A']) self.executor_server.release('A') self.waitUntilSettled() self.assertEqual(sorted(b.name for b in self.builds), ['B', 'C']) self.executor_server.release('B') self.waitUntilSettled() self.assertEqual(sorted(b.name for b in self.builds), ['C', 'D']) self.executor_server.release('D') self.waitUntilSettled() self.assertEqual([b.name for b in self.builds], ['C']) self.executor_server.release('C') self.waitUntilSettled() self.assertEqual(len(self.builds), 0) self.executor_server.hold_jobs_in_build = False self.executor_server.release() self.waitUntilSettled() self.assertEqual(len(self.builds), 0) self.assertEqual(len(self.history), 4) self.assertEqual(change.data['status'], 'NEW') self.assertEqual(change.reported, 2) @simple_layout('layouts/soft-dependencies-error.yaml') def test_soft_dependencies_error(self): A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) self.waitUntilSettled() self.assertHistory([]) self.assertEqual(len(A.messages), 1) self.assertTrue('Job project-merge not defined' in A.messages[0]) self.log.info(A.messages) @simple_layout('layouts/soft-dependencies.yaml') def test_soft_dependencies(self): A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) self.waitUntilSettled() self.assertHistory([ dict(name='deploy', result='SUCCESS', changes='1,1'), ], ordered=False) @simple_layout('layouts/not-skip-when-reenqueue.yaml') def test_child_with_soft_dependency_should_not_skip(self): A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') self.executor_server.hold_jobs_in_build = True self.executor_server.returnData( 'grand-parent', A, {'zuul': {'child_jobs': ['parent2']} } ) self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) self.waitUntilSettled() self.executor_server.release('grand-parent') self.waitUntilSettled() # grand-parent success, parent1 skipped, parent2 running self.assertHistory([ dict(name='grand-parent', result='SUCCESS', changes='1,1'), ], ordered=False) self.assertBuilds([dict(name='parent2')]) # Reconfigure to trigger a re-enqueue, this should not cause job # 'child' to be skipped because parent1 was skipped self.scheds.execute(lambda app: app.sched.reconfigure(app.config)) self.executor_server.release('parent1') self.executor_server.release('parent2') self.waitUntilSettled() # grand-parent success, parent1 skipped, parent2 success, child running self.assertHistory([ dict(name='grand-parent', result='SUCCESS', changes='1,1'), dict(name='parent2', result='SUCCESS', changes='1,1'), ], ordered=False) self.assertBuilds([dict(name='child')]) self.executor_server.release('child') self.waitUntilSettled() # grand-parent success, parent1 skipped, parent2 success, child success self.assertHistory([ dict(name='grand-parent', result='SUCCESS', changes='1,1'), dict(name='parent2', result='SUCCESS', changes='1,1'), dict(name='child', result='SUCCESS', changes='1,1'), ], ordered=False) self.assertBuilds([]) @simple_layout('layouts/soft-dependencies.yaml') def test_soft_dependencies_failure(self): file_dict = {'main.c': 'test'} A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A', files=file_dict) self.executor_server.failJob('build', A) self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) self.waitUntilSettled() self.assertHistory([ dict(name='build', result='FAILURE', changes='1,1'), ], ordered=False) self.assertIn('Skipped due to failed job build', A.messages[0]) class TestDuplicatePipeline(ZuulTestCase): tenant_config_file = 'config/duplicate-pipeline/main.yaml' def test_duplicate_pipelines(self): "Test that a change matching multiple pipelines works" A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') self.fake_gerrit.addEvent(A.getChangeRestoredEvent()) self.waitUntilSettled() self.assertHistory([ dict(name='project-test1', result='SUCCESS', changes='1,1', pipeline='dup1'), dict(name='project-test1', result='SUCCESS', changes='1,1', pipeline='dup2'), ], ordered=False) self.assertEqual(len(A.messages), 2) if 'dup1' in A.messages[0]: self.assertIn('dup1', A.messages[0]) self.assertNotIn('dup2', A.messages[0]) self.assertIn('project-test1', A.messages[0]) self.assertIn('dup2', A.messages[1]) self.assertNotIn('dup1', A.messages[1]) self.assertIn('project-test1', A.messages[1]) else: self.assertIn('dup1', A.messages[1]) self.assertNotIn('dup2', A.messages[1]) self.assertIn('project-test1', A.messages[1]) self.assertIn('dup2', A.messages[0]) self.assertNotIn('dup1', A.messages[0]) self.assertIn('project-test1', A.messages[0]) class TestSchedulerRegexProject(ZuulTestCase): tenant_config_file = 'config/regex-project/main.yaml' def test_regex_project(self): "Test that changes are tested in parallel and merged in series" A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B') C = self.fake_gerrit.addFakeChange('org/project2', 'master', 'C') self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1)) self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1)) self.waitUntilSettled() # We expect the following builds: # - 1 for org/project # - 3 for org/project1 # - 3 for org/project2 self.assertEqual(len(self.history), 7) self.assertEqual(A.reported, 1) self.assertEqual(B.reported, 1) self.assertEqual(C.reported, 1) self.assertHistory([ dict(name='project-test', result='SUCCESS', changes='1,1'), dict(name='project-test1', result='SUCCESS', changes='2,1'), dict(name='project-common-test', result='SUCCESS', changes='2,1'), dict(name='project-common-test-canonical', result='SUCCESS', changes='2,1'), dict(name='project-test2', result='SUCCESS', changes='3,1'), dict(name='project-common-test', result='SUCCESS', changes='3,1'), dict(name='project-common-test-canonical', result='SUCCESS', changes='3,1'), ], ordered=False) class TestSchedulerTemplatedProject(ZuulTestCase): tenant_config_file = 'config/templated-project/main.yaml' def test_job_from_templates_executed(self): "Test whether a job generated via a template can be executed" A = self.fake_gerrit.addFakeChange( 'org/templated-project', 'master', 'A') self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) self.waitUntilSettled() self.assertEqual(self.getJobFromHistory('project-test1').result, 'SUCCESS') self.assertEqual(self.getJobFromHistory('project-test2').result, 'SUCCESS') def test_layered_templates(self): "Test whether a job generated via a template can be executed" A = self.fake_gerrit.addFakeChange( 'org/layered-project', 'master', 'A') self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) self.waitUntilSettled() self.assertEqual(self.getJobFromHistory('project-test1').result, 'SUCCESS') self.assertEqual(self.getJobFromHistory('project-test2').result, 'SUCCESS') self.assertEqual(self.getJobFromHistory('layered-project-test3' ).result, 'SUCCESS') self.assertEqual(self.getJobFromHistory('layered-project-test4' ).result, 'SUCCESS') self.assertEqual(self.getJobFromHistory('layered-project-foo-test5' ).result, 'SUCCESS') self.assertEqual(self.getJobFromHistory('project-test6').result, 'SUCCESS') def test_unimplied_branch_matchers(self): # This tests that there are no implied branch matchers added # to project templates in unbranched projects. self.create_branch('org/layered-project', 'stable') self.fake_gerrit.addEvent( self.fake_gerrit.getFakeBranchCreatedEvent( 'org/layered-project', 'stable')) self.waitUntilSettled() A = self.fake_gerrit.addFakeChange( 'org/layered-project', 'stable', 'A') self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) self.waitUntilSettled() self.assertEqual(self.getJobFromHistory('project-test1').result, 'SUCCESS') self.log.info( self.getJobFromHistory('project-test1'). parameters['zuul']['_inheritance_path']) def test_implied_branch_matchers(self): # This tests that there is an implied branch matcher when a # template is used on an in-repo project pipeline definition. self.create_branch('untrusted-config', 'stable') self.fake_gerrit.addEvent( self.fake_gerrit.getFakeBranchCreatedEvent( 'untrusted-config', 'stable')) self.waitUntilSettled() A = self.fake_gerrit.addFakeChange( 'untrusted-config', 'stable', 'A') self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) self.waitUntilSettled() self.assertEqual(self.getJobFromHistory('project-test1').result, 'SUCCESS') self.log.info( self.getJobFromHistory('project-test1'). parameters['zuul']['_inheritance_path']) # Now create a new branch named stable-foo and change the project # pipeline self.create_branch('untrusted-config', 'stable-foo') self.fake_gerrit.addEvent( self.fake_gerrit.getFakeBranchCreatedEvent( 'untrusted-config', 'stable-foo')) self.waitUntilSettled() in_repo_conf = textwrap.dedent( """ - project: name: untrusted-config templates: - test-three-and-four check: jobs: - project-test7 """) file_dict = {'zuul.d/project.yaml': in_repo_conf} B = self.fake_gerrit.addFakeChange('untrusted-config', 'stable-foo', 'B', files=file_dict) self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1)) self.waitUntilSettled() self.assertHistory([ dict(name='project-test1', result='SUCCESS', changes='1,1'), dict(name='project-test2', result='SUCCESS', changes='1,1'), dict(name='project-test7', result='SUCCESS', changes='2,1'), dict(name='layered-project-test3', result='SUCCESS', changes='2,1'), dict(name='layered-project-test4', result='SUCCESS', changes='2,1'), ], ordered=False) # Inheritance path should not contain items from branch stable # This tests that not only is it the case that the stable # branch project-template did not apply, but also that the # stable branch definitions of the project-test7 did not apply # (since the job definitions also have implied branch # matchers). job = self.getJobFromHistory('project-test7', branch='stable-foo') inheritance_path = job.parameters['zuul']['_inheritance_path'] self.assertEqual(len(inheritance_path), 4) stable_items = [x for x in inheritance_path if 'untrusted-config/zuul.d/jobs.yaml@stable#' in x] self.assertEqual(len(stable_items), 0) class TestSchedulerMerges(ZuulTestCase): tenant_config_file = 'config/merges/main.yaml' def _test_project_merge_mode(self, mode): self.executor_server.keep_jobdir = False project = 'org/project-%s' % mode self.executor_server.hold_jobs_in_build = True A = self.fake_gerrit.addFakeChange(project, 'master', 'A') B = self.fake_gerrit.addFakeChange(project, 'master', 'B') C = self.fake_gerrit.addFakeChange(project, 'master', 'C') A.addApproval('Code-Review', 2) B.addApproval('Code-Review', 2) C.addApproval('Code-Review', 2) self.fake_gerrit.addEvent(A.addApproval('Approved', 1)) self.fake_gerrit.addEvent(B.addApproval('Approved', 1)) self.fake_gerrit.addEvent(C.addApproval('Approved', 1)) self.waitUntilSettled() build = self.builds[-1] path = os.path.join(build.jobdir.src_root, 'review.example.com', project) repo = git.Repo(path) repo_messages = [c.message.strip() for c in repo.iter_commits()] repo_messages.reverse() self.executor_server.hold_jobs_in_build = False self.executor_server.release() self.waitUntilSettled() return repo_messages def _test_merge(self, mode): us_path = 'file://' + os.path.join( self.upstream_root, 'org/project-%s' % mode) expected_messages = [ 'initial commit', 'add content from fixture', # the intermediate commits order is nondeterministic "Merge commit 'refs/changes/02/2/1' of %s into HEAD" % us_path, "Merge commit 'refs/changes/03/3/1' of %s into HEAD" % us_path, ] result = self._test_project_merge_mode(mode) self.assertEqual(result[:2], expected_messages[:2]) self.assertEqual(result[-2:], expected_messages[-2:]) def test_project_merge_mode_merge(self): self._test_merge('merge') def test_project_merge_mode_merge_resolve(self): self._test_merge('merge-resolve') def test_project_merge_mode_cherrypick(self): expected_messages = [ 'initial commit', 'add content from fixture', 'A-1', 'B-1', 'C-1'] result = self._test_project_merge_mode('cherry-pick') self.assertEqual(result, expected_messages) def test_project_merge_mode_cherrypick_redundant(self): # A redundant commit (that is, one that has already been applied to the # working tree) should be skipped self.executor_server.keep_jobdir = False project = 'org/project-cherry-pick' files = { "foo.txt": "ABC", } A = self.fake_gerrit.addFakeChange(project, 'master', 'A', files=files) A.addApproval('Code-Review', 2) self.fake_gerrit.addEvent(A.addApproval('Approved', 1)) self.waitUntilSettled() self.executor_server.hold_jobs_in_build = True B = self.fake_gerrit.addFakeChange(project, 'master', 'B', files=files) B.addApproval('Code-Review', 2) self.fake_gerrit.addEvent(B.addApproval('Approved', 1)) self.waitUntilSettled() build = self.builds[-1] path = os.path.join(build.jobdir.src_root, 'review.example.com', project) repo = git.Repo(path) repo_messages = [c.message.strip() for c in repo.iter_commits()] repo_messages.reverse() self.executor_server.hold_jobs_in_build = False self.executor_server.release() self.waitUntilSettled() expected_messages = [ 'initial commit', 'add content from fixture', 'A-1', ] self.assertHistory([ dict(name='project-test1', result='SUCCESS', changes='1,1'), dict(name='project-test1', result='SUCCESS', changes='2,1'), ]) self.assertEqual(A.data['status'], 'MERGED') self.assertEqual(B.data['status'], 'MERGED') self.assertEqual(repo_messages, expected_messages) def test_project_merge_mode_cherrypick_empty(self): # An empty commit (that is, one that doesn't modify any files) should # be preserved self.executor_server.keep_jobdir = False project = 'org/project-cherry-pick' self.executor_server.hold_jobs_in_build = True A = self.fake_gerrit.addFakeChange(project, 'master', 'A', empty=True) A.addApproval('Code-Review', 2) self.fake_gerrit.addEvent(A.addApproval('Approved', 1)) self.waitUntilSettled() build = self.builds[-1] path = os.path.join(build.jobdir.src_root, 'review.example.com', project) repo = git.Repo(path) repo_messages = [c.message.strip() for c in repo.iter_commits()] repo_messages.reverse() changed_files = list(repo.commit("HEAD").diff(repo.commit("HEAD~1"))) self.assertEqual(changed_files, []) self.executor_server.hold_jobs_in_build = False self.executor_server.release() self.waitUntilSettled() expected_messages = [ 'initial commit', 'add content from fixture', 'A-1', ] self.assertHistory([ dict(name='project-test1', result='SUCCESS', changes='1,1'), ]) self.assertEqual(A.data['status'], 'MERGED') self.assertEqual(repo_messages, expected_messages) def test_project_merge_mode_cherrypick_branch_merge(self): "Test that branches can be merged together in cherry-pick mode" self.create_branch('org/project-merge-branches', 'mp') self.fake_gerrit.addEvent( self.fake_gerrit.getFakeBranchCreatedEvent( 'org/project-merge-branches', 'mp')) self.waitUntilSettled() path = os.path.join(self.upstream_root, 'org/project-merge-branches') repo = git.Repo(path) master_sha = repo.heads.master.commit.hexsha mp_sha = repo.heads.mp.commit.hexsha self.executor_server.hold_jobs_in_build = True M = self.fake_gerrit.addFakeChange( 'org/project-merge-branches', 'master', 'M', merge_parents=[ master_sha, mp_sha, ]) M.addApproval('Code-Review', 2) self.fake_gerrit.addEvent(M.addApproval('Approved', 1)) self.waitUntilSettled() self.executor_server.release('.*-merge') self.waitUntilSettled() build = self.builds[-1] self.assertEqual(build.parameters['zuul']['branch'], 'master') path = os.path.join(build.jobdir.src_root, 'review.example.com', "org/project-merge-branches") repo = git.Repo(path) repo_messages = [c.message.strip() for c in repo.iter_commits()] repo_messages.reverse() correct_messages = [ 'initial commit', 'add content from fixture', 'mp commit', 'M-1'] self.assertEqual(repo_messages, correct_messages) self.executor_server.hold_jobs_in_build = False self.executor_server.release() self.waitUntilSettled() def test_merge_branch(self): "Test that the right commits are on alternate branches" self.create_branch('org/project-merge-branches', 'mp') self.fake_gerrit.addEvent( self.fake_gerrit.getFakeBranchCreatedEvent( 'org/project-merge-branches', 'mp')) self.waitUntilSettled() self.executor_server.hold_jobs_in_build = True A = self.fake_gerrit.addFakeChange( 'org/project-merge-branches', 'mp', 'A') B = self.fake_gerrit.addFakeChange( 'org/project-merge-branches', 'mp', 'B') C = self.fake_gerrit.addFakeChange( 'org/project-merge-branches', 'mp', 'C') A.addApproval('Code-Review', 2) B.addApproval('Code-Review', 2) C.addApproval('Code-Review', 2) self.fake_gerrit.addEvent(A.addApproval('Approved', 1)) self.fake_gerrit.addEvent(B.addApproval('Approved', 1)) self.fake_gerrit.addEvent(C.addApproval('Approved', 1)) self.waitUntilSettled() self.executor_server.release('.*-merge') self.waitUntilSettled() self.executor_server.release('.*-merge') self.waitUntilSettled() self.executor_server.release('.*-merge') self.waitUntilSettled() build = self.builds[-1] self.assertEqual(build.parameters['zuul']['branch'], 'mp') path = os.path.join(build.jobdir.src_root, 'review.example.com', 'org/project-merge-branches') repo = git.Repo(path) repo_messages = [c.message.strip() for c in repo.iter_commits()] repo_messages.reverse() correct_messages = [ 'initial commit', 'add content from fixture', 'mp commit', 'A-1', 'B-1', 'C-1'] self.assertEqual(repo_messages, correct_messages) self.executor_server.hold_jobs_in_build = False self.executor_server.release() self.waitUntilSettled() def test_merge_multi_branch(self): "Test that dependent changes on multiple branches are merged" self.create_branch('org/project-merge-branches', 'mp') self.fake_gerrit.addEvent( self.fake_gerrit.getFakeBranchCreatedEvent( 'org/project-merge-branches', 'mp')) self.waitUntilSettled() self.executor_server.hold_jobs_in_build = True A = self.fake_gerrit.addFakeChange( 'org/project-merge-branches', 'master', 'A') B = self.fake_gerrit.addFakeChange( 'org/project-merge-branches', 'mp', 'B') C = self.fake_gerrit.addFakeChange( 'org/project-merge-branches', 'master', 'C') A.addApproval('Code-Review', 2) B.addApproval('Code-Review', 2) C.addApproval('Code-Review', 2) self.fake_gerrit.addEvent(A.addApproval('Approved', 1)) self.fake_gerrit.addEvent(B.addApproval('Approved', 1)) self.fake_gerrit.addEvent(C.addApproval('Approved', 1)) self.waitUntilSettled() job_A = None for job in self.builds: if 'project-merge' in job.name: job_A = job path = os.path.join(job_A.jobdir.src_root, 'review.example.com', 'org/project-merge-branches') repo = git.Repo(path) repo_messages = [c.message.strip() for c in repo.iter_commits()] repo_messages.reverse() correct_messages = [ 'initial commit', 'add content from fixture', 'A-1'] self.assertEqual(repo_messages, correct_messages) self.executor_server.release('.*-merge') self.waitUntilSettled() job_B = None for job in self.builds: if 'project-merge' in job.name: job_B = job path = os.path.join(job_B.jobdir.src_root, 'review.example.com', 'org/project-merge-branches') repo = git.Repo(path) repo_messages = [c.message.strip() for c in repo.iter_commits()] repo_messages.reverse() correct_messages = [ 'initial commit', 'add content from fixture', 'mp commit', 'B-1'] self.assertEqual(repo_messages, correct_messages) self.executor_server.release('.*-merge') self.waitUntilSettled() job_C = None for job in self.builds: if 'project-merge' in job.name: job_C = job path = os.path.join(job_C.jobdir.src_root, 'review.example.com', 'org/project-merge-branches') repo = git.Repo(path) repo_messages = [c.message.strip() for c in repo.iter_commits()] repo_messages.reverse() correct_messages = [ 'initial commit', 'add content from fixture', 'A-1', 'C-1'] # Ensure the right commits are in the history for this ref self.assertEqual(repo_messages, correct_messages) self.executor_server.hold_jobs_in_build = False self.executor_server.release() self.waitUntilSettled() class TestSemaphore(ZuulTestCase): tenant_config_file = 'config/semaphore/main.yaml' def test_semaphore_one(self): "Test semaphores with max=1 (mutex)" tenant = self.scheds.first.sched.abide.tenants.get('tenant-one') self.executor_server.hold_jobs_in_build = True # Pause nodepool so we can check the ordering of getting the nodes # and aquiring the semaphore. self.fake_nodepool.paused = True A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B') self.assertEqual( len(tenant.semaphore_handler.semaphoreHolders("test-semaphore")), 0) self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1)) self.waitUntilSettled() status = tenant.layout.pipelines["check"].formatStatusJSON() jobs = status["change_queues"][0]["heads"][0][0]["jobs"] self.assertEqual(jobs[0]["waiting_status"], 'node request: 200-0000000000') self.assertEqual(jobs[1]["waiting_status"], 'node request: 200-0000000001') self.assertEqual(jobs[2]["waiting_status"], 'semaphores: test-semaphore') # By default we first lock the semaphore and then get the nodes # so at this point the semaphore needs to be aquired. self.assertEqual( len(tenant.semaphore_handler.semaphoreHolders("test-semaphore")), 1) self.fake_nodepool.paused = False self.waitUntilSettled() self.assertEqual(len(self.builds), 3) self.assertEqual(self.builds[0].name, 'project-test1') self.assertEqual(self.builds[1].name, 'semaphore-one-test1') self.assertEqual(self.builds[2].name, 'project-test1') self.executor_server.release('semaphore-one-test1') self.waitUntilSettled() self.assertEqual(len(self.builds), 3) self.assertEqual(self.builds[0].name, 'project-test1') self.assertEqual(self.builds[1].name, 'project-test1') self.assertEqual(self.builds[2].name, 'semaphore-one-test2') self.assertEqual( len(tenant.semaphore_handler.semaphoreHolders("test-semaphore")), 1) self.executor_server.release('semaphore-one-test2') self.waitUntilSettled() self.assertEqual(len(self.builds), 3) self.assertEqual(self.builds[0].name, 'project-test1') self.assertEqual(self.builds[1].name, 'project-test1') self.assertEqual(self.builds[2].name, 'semaphore-one-test1') self.assertEqual( len(tenant.semaphore_handler.semaphoreHolders("test-semaphore")), 1) self.executor_server.release('semaphore-one-test1') self.waitUntilSettled() self.assertEqual(len(self.builds), 3) self.assertEqual(self.builds[0].name, 'project-test1') self.assertEqual(self.builds[1].name, 'project-test1') self.assertEqual(self.builds[2].name, 'semaphore-one-test2') self.assertEqual( len(tenant.semaphore_handler.semaphoreHolders("test-semaphore")), 1) self.executor_server.release('semaphore-one-test2') self.waitUntilSettled() self.assertEqual(len(self.builds), 2) self.assertEqual(self.builds[0].name, 'project-test1') self.assertEqual(self.builds[1].name, 'project-test1') self.assertEqual( len(tenant.semaphore_handler.semaphoreHolders("test-semaphore")), 0) self.executor_server.hold_jobs_in_build = False self.executor_server.release() self.waitUntilSettled() self.assertEqual(len(self.builds), 0) self.assertEqual(A.reported, 1) self.assertEqual(B.reported, 1) self.assertEqual( len(tenant.semaphore_handler.semaphoreHolders("test-semaphore")), 0) self.assertReportedStat( 'zuul.tenant.tenant-one.semaphore.test-semaphore.holders', value='1', kind='g') self.assertReportedStat( 'zuul.tenant.tenant-one.semaphore.test-semaphore.holders', value='0', kind='g') def test_semaphore_two(self): "Test semaphores with max>1" tenant = self.scheds.first.sched.abide.tenants.get('tenant-one') self.executor_server.hold_jobs_in_build = True A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A') B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B') self.assertEqual( len(tenant.semaphore_handler.semaphoreHolders( "test-semaphore-two")), 0) self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1)) self.waitUntilSettled() self.assertEqual(len(self.builds), 4) self.assertEqual(self.builds[0].name, 'project-test1') self.assertEqual(self.builds[1].name, 'semaphore-two-test1') self.assertEqual(self.builds[2].name, 'semaphore-two-test2') self.assertEqual(self.builds[3].name, 'project-test1') self.assertEqual( len(tenant.semaphore_handler.semaphoreHolders( "test-semaphore-two")), 2) self.executor_server.release('semaphore-two-test1') self.waitUntilSettled() self.assertEqual(len(self.builds), 4) self.assertEqual(self.builds[0].name, 'project-test1') self.assertEqual(self.builds[1].name, 'semaphore-two-test2') self.assertEqual(self.builds[2].name, 'project-test1') self.assertEqual(self.builds[3].name, 'semaphore-two-test1') self.assertEqual( len(tenant.semaphore_handler.semaphoreHolders( "test-semaphore-two")), 2) self.executor_server.release('semaphore-two-test2') self.waitUntilSettled() self.assertEqual(len(self.builds), 4) self.assertEqual(self.builds[0].name, 'project-test1') self.assertEqual(self.builds[1].name, 'project-test1') self.assertEqual(self.builds[2].name, 'semaphore-two-test1') self.assertEqual(self.builds[3].name, 'semaphore-two-test2') self.assertEqual( len(tenant.semaphore_handler.semaphoreHolders( "test-semaphore-two")), 2) self.executor_server.release('semaphore-two-test1') self.waitUntilSettled() self.assertEqual(len(self.builds), 3) self.assertEqual(self.builds[0].name, 'project-test1') self.assertEqual(self.builds[1].name, 'project-test1') self.assertEqual(self.builds[2].name, 'semaphore-two-test2') self.assertEqual( len(tenant.semaphore_handler.semaphoreHolders( "test-semaphore-two")), 1) self.executor_server.release('semaphore-two-test2') self.waitUntilSettled() self.assertEqual(len(self.builds), 2) self.assertEqual(self.builds[0].name, 'project-test1') self.assertEqual(self.builds[1].name, 'project-test1') self.assertEqual( len(tenant.semaphore_handler.semaphoreHolders( "test-semaphore-two")), 0) self.executor_server.hold_jobs_in_build = False self.executor_server.release() self.waitUntilSettled() self.assertEqual(len(self.builds), 0) self.assertEqual(A.reported, 1) self.assertEqual(B.reported, 1) def test_semaphore_node_failure(self): "Test semaphore and node failure" tenant = self.scheds.first.sched.abide.tenants.get('tenant-one') # Pause nodepool so we can fail the node request later self.fake_nodepool.pause() A = self.fake_gerrit.addFakeChange('org/project2', 'master', 'A') self.assertEqual( len(tenant.semaphore_handler.semaphoreHolders("test-semaphore")), 0) self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) self.waitUntilSettled() # By default we first lock the semaphore and then get the nodes # so at this point the semaphore needs to be aquired. self.assertEqual( len(tenant.semaphore_handler.semaphoreHolders("test-semaphore")), 1) # Fail the node request and unpause req = self.fake_nodepool.getNodeRequests()[0] self.fake_nodepool.addFailRequest(req) self.fake_nodepool.unpause() self.waitUntilSettled() # At this point the job that holds the semaphore failed with # node_failure and the semaphore must be released. self.assertEqual( len(tenant.semaphore_handler.semaphoreHolders("test-semaphore")), 0) self.assertEquals(1, A.reported) self.assertTrue(re.search('semaphore-one-test3 .* NODE_FAILURE', A.messages[0])) def test_semaphore_resources_first(self): "Test semaphores with max=1 (mutex) and get resources first" tenant = self.scheds.first.sched.abide.tenants.get('tenant-one') self.executor_server.hold_jobs_in_build = True # Pause nodepool so we can check the ordering of getting the nodes # and aquiring the semaphore. self.fake_nodepool.paused = True A = self.fake_gerrit.addFakeChange('org/project3', 'master', 'A') B = self.fake_gerrit.addFakeChange('org/project3', 'master', 'B') self.assertEqual( len(tenant.semaphore_handler.semaphoreHolders("test-semaphore")), 0) self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1)) self.waitUntilSettled() # Here we first get the resources and then lock the semaphore # so at this point the semaphore should not be aquired. self.assertEqual( len(tenant.semaphore_handler.semaphoreHolders("test-semaphore")), 0) self.fake_nodepool.paused = False self.waitUntilSettled() self.assertEqual(len(self.builds), 3) self.assertEqual(self.builds[0].name, 'project-test1') self.assertEqual(self.builds[1].name, 'semaphore-one-test1-resources-first') self.assertEqual(self.builds[2].name, 'project-test1') self.executor_server.release('semaphore-one-test1') self.waitUntilSettled() self.assertEqual(len(self.builds), 3) self.assertEqual(self.builds[0].name, 'project-test1') self.assertEqual(self.builds[1].name, 'project-test1') self.assertEqual(self.builds[2].name, 'semaphore-one-test2-resources-first') self.assertEqual( len(tenant.semaphore_handler.semaphoreHolders("test-semaphore")), 1) self.executor_server.hold_jobs_in_build = False self.executor_server.release() self.waitUntilSettled() def test_semaphore_resources_first_node_failure(self): "Test semaphore and node failure" tenant = self.scheds.first.sched.abide.tenants.get('tenant-one') # Pause nodepool so we can fail the node request later self.fake_nodepool.pause() A = self.fake_gerrit.addFakeChange('org/project4', 'master', 'A') self.assertEqual( len(tenant.semaphore_handler.semaphoreHolders("test-semaphore")), 0) self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) self.waitUntilSettled() # With resources first we first get the nodes so at this point the # semaphore must not be aquired. self.assertEqual(len(tenant.semaphore_handler.semaphoreHolders( "test-semaphore")), 0) # Fail the node request and unpause req = self.fake_nodepool.getNodeRequests()[0] self.fake_nodepool.addFailRequest(req) self.fake_nodepool.unpause() self.waitUntilSettled() # At this point the job should never have acuired a semaphore so check # that it still has not locked a semaphore. self.assertEqual(len(tenant.semaphore_handler.semaphoreHolders( "test-semaphore")), 0) self.assertEquals(1, A.reported) self.assertTrue( re.search('semaphore-one-test1-resources-first .* NODE_FAILURE', A.messages[0])) def test_semaphore_zk_error(self): "Test semaphore release with zk error" tenant = self.scheds.first.sched.abide.tenants.get('tenant-one') A = self.fake_gerrit.addFakeChange('org/project2', 'master', 'A') self.assertEqual( len(tenant.semaphore_handler.semaphoreHolders("test-semaphore")), 0) # Simulate a single zk error in useNodeSet orig_useNodeSet = self.scheds.first.sched.nodepool.useNodeSet def broken_use_nodeset(nodeset, tenant_name, project_name): # restore original useNodeSet self.scheds.first.sched.nodepool.useNodeSet = orig_useNodeSet raise NoNodeError() self.scheds.first.sched.nodepool.useNodeSet = broken_use_nodeset self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) self.waitUntilSettled() # The semaphore should be released self.assertEqual( len(tenant.semaphore_handler.semaphoreHolders("test-semaphore")), 0) # cleanup the queue self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) self.waitUntilSettled() def test_semaphore_abandon(self): "Test abandon with job semaphores" self.executor_server.hold_jobs_in_build = True tenant = self.scheds.first.sched.abide.tenants.get('tenant-one') check_pipeline = tenant.layout.pipelines['check'] A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') self.assertEqual( len(tenant.semaphore_handler.semaphoreHolders("test-semaphore")), 0) self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) self.waitUntilSettled() self.assertEqual( len(tenant.semaphore_handler.semaphoreHolders("test-semaphore")), 1) self.fake_gerrit.addEvent(A.getChangeAbandonedEvent()) self.waitUntilSettled() # The check pipeline should be empty items = check_pipeline.getAllItems() self.assertEqual(len(items), 0) # The semaphore should be released self.assertEqual( len(tenant.semaphore_handler.semaphoreHolders("test-semaphore")), 0) self.executor_server.hold_jobs_in_build = False self.executor_server.release() self.waitUntilSettled() def test_semaphore_abandon_pending_node_request(self): "Test abandon with job semaphores and pending node request" self.executor_server.hold_jobs_in_build = True # Pause nodepool so we can check the ordering of getting the nodes # and aquiring the semaphore. self.fake_nodepool.paused = True tenant = self.scheds.first.sched.abide.tenants.get('tenant-one') check_pipeline = tenant.layout.pipelines['check'] A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') self.assertEqual( len(tenant.semaphore_handler.semaphoreHolders("test-semaphore")), 0) self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) self.waitUntilSettled() self.assertEqual( len(tenant.semaphore_handler.semaphoreHolders("test-semaphore")), 1) self.fake_gerrit.addEvent(A.getChangeAbandonedEvent()) self.waitUntilSettled() # The check pipeline should be empty items = check_pipeline.getAllItems() self.assertEqual(len(items), 0) # The semaphore should be released self.assertEqual( len(tenant.semaphore_handler.semaphoreHolders("test-semaphore")), 0) self.executor_server.hold_jobs_in_build = False self.fake_nodepool.paused = False self.executor_server.release() self.waitUntilSettled() def test_semaphore_abandon_pending_execution(self): "Test abandon with job semaphores and pending job execution" # Pause the executor so it doesn't take any jobs. self.executor_server.pause() # Start merger as the paused executor won't take merge jobs. self._startMerger() # Pause nodepool so we can wait on the node requests and fulfill them # in a controlled manner. self.fake_nodepool.paused = True tenant = self.scheds.first.sched.abide.tenants.get('tenant-one') check_pipeline = tenant.layout.pipelines['check'] A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') self.assertEqual( len(tenant.semaphore_handler.semaphoreHolders("test-semaphore")), 0) self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) self.waitUntilSettled() reqs = list(self.scheds.first.sched.nodepool.getNodeRequests()) self.assertEqual(len(reqs), 2) # Now unpause nodepool to fulfill the node requests. We cannot use # waitUntilSettled here because the executor is paused. self.fake_nodepool.paused = False for _ in iterate_timeout(30, 'fulfill node requests'): reqs = [ r for r in self.scheds.first.sched.nodepool.getNodeRequests() if r.state != zuul.model.STATE_FULFILLED ] if len(reqs) == 0: break self.assertEqual( len(tenant.semaphore_handler.semaphoreHolders("test-semaphore")), 1) self.fake_gerrit.addEvent(A.getChangeAbandonedEvent()) self.waitUntilSettled() # The check pipeline should be empty items = check_pipeline.getAllItems() self.assertEqual(len(items), 0) # The semaphore should be released self.assertEqual( len(tenant.semaphore_handler.semaphoreHolders("test-semaphore")), 0) self.executor_server.release() self.waitUntilSettled() def test_semaphore_new_patchset(self): "Test new patchset with job semaphores" self.executor_server.hold_jobs_in_build = True tenant = self.scheds.first.sched.abide.tenants.get('tenant-one') check_pipeline = tenant.layout.pipelines['check'] A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') self.assertEqual( len(tenant.semaphore_handler.semaphoreHolders("test-semaphore")), 0) self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) self.waitUntilSettled() self.assertEqual( len(tenant.semaphore_handler.semaphoreHolders("test-semaphore")), 1) A.addPatchset() self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(2)) self.waitUntilSettled() self.assertEqual( len(tenant.semaphore_handler.semaphoreHolders("test-semaphore")), 1) items = check_pipeline.getAllItems() self.assertEqual(items[0].changes[0].number, '1') self.assertEqual(items[0].changes[0].patchset, '2') self.assertTrue(items[0].live) self.executor_server.hold_jobs_in_build = False self.executor_server.release() self.waitUntilSettled() # The semaphore should be released self.assertEqual( len(tenant.semaphore_handler.semaphoreHolders("test-semaphore")), 0) def test_semaphore_reconfigure(self): "Test reconfigure with job semaphores" self.executor_server.hold_jobs_in_build = True tenant = self.scheds.first.sched.abide.tenants.get('tenant-one') A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') self.assertEqual( len(tenant.semaphore_handler.semaphoreHolders("test-semaphore")), 0) self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) self.waitUntilSettled() self.assertEqual( len(tenant.semaphore_handler.semaphoreHolders("test-semaphore")), 1) # reconfigure without layout change self.scheds.execute(lambda app: app.sched.reconfigure(app.config)) self.waitUntilSettled() tenant = self.scheds.first.sched.abide.tenants.get('tenant-one') # semaphore still must be held self.assertEqual( len(tenant.semaphore_handler.semaphoreHolders("test-semaphore")), 1) # remove the pipeline self.commitConfigUpdate( 'common-config', 'config/semaphore/zuul-reconfiguration.yaml') self.scheds.execute(lambda app: app.sched.reconfigure(app.config)) self.waitUntilSettled() tenant = self.scheds.first.sched.abide.tenants.get('tenant-one') self.executor_server.release('project-test1') self.waitUntilSettled() # There should be no builds anymore self.assertEqual(len(self.builds), 0) # The semaphore should be released self.assertEqual( len(tenant.semaphore_handler.semaphoreHolders("test-semaphore")), 0) def test_semaphore_handler_cleanup(self): "Test the semaphore handler leak cleanup" self.executor_server.hold_jobs_in_build = True tenant = self.scheds.first.sched.abide.tenants.get('tenant-one') A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') self.assertEqual( len(tenant.semaphore_handler.semaphoreHolders("test-semaphore")), 0) self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) self.waitUntilSettled() self.assertEqual( len(tenant.semaphore_handler.semaphoreHolders("test-semaphore")), 1) # Save some variables for later use while the job is running check_pipeline = tenant.layout.pipelines['check'] item = check_pipeline.getAllItems()[0] job = list(filter(lambda j: j.name == 'semaphore-one-test1', item.getJobs()))[0] tenant.semaphore_handler.cleanupLeaks() # Nothing has leaked; our handle should be present. self.assertEqual( len(tenant.semaphore_handler.semaphoreHolders("test-semaphore")), 1) self.executor_server.hold_jobs_in_build = False self.executor_server.release() self.waitUntilSettled() # Make sure the semaphore is released normally self.assertEqual( len(tenant.semaphore_handler.semaphoreHolders("test-semaphore")), 0) # Use our previously saved data to simulate a leaked semaphore tenant.semaphore_handler.acquire(item, job, False) self.assertEqual( len(tenant.semaphore_handler.semaphoreHolders("test-semaphore")), 1) tenant.semaphore_handler.cleanupLeaks() # Make sure the leaked semaphore is cleaned up self.assertEqual( len(tenant.semaphore_handler.semaphoreHolders("test-semaphore")), 0) @simple_layout('layouts/multiple-semaphores.yaml') def test_multiple_semaphores(self): # Test a job with multiple semaphores self.executor_server.hold_jobs_in_build = True A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A') self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) self.waitUntilSettled() # One job should be running, and hold sem1 self.assertBuilds([dict(name='job1')]) B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B') self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1)) self.waitUntilSettled() # Job2 requires sem1 and sem2; it hasn't started because job # is still holding sem1. self.assertBuilds([dict(name='job1')]) self.executor_server.release('job1') self.waitUntilSettled() # Job1 is finished, so job2 can acquire both semaphores. self.assertBuilds([dict(name='job2')]) self.executor_server.release() self.waitUntilSettled() self.assertHistory([ dict(name='job1', result='SUCCESS', changes='1,1'), dict(name='job2', result='SUCCESS', changes='2,1'), ]) # TODO(corvus): Consider a version of this test which launches # 2 jobs with the same multiple-semaphore requirements # simultaneously to test the behavior with contention (at # least one should be able to start on each pass through the # loop). @simple_layout('layouts/semaphore-multi-pipeline.yaml') def test_semaphore_multi_pipeline(self): "Test semaphores in multiple pipelines" tenant = self.scheds.first.sched.abide.tenants.get('tenant-one') self.executor_server.hold_jobs_in_build = True A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') self.assertEqual( len(tenant.semaphore_handler.semaphoreHolders("test-semaphore")), 0) self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) self.waitUntilSettled() self.assertEqual(len(self.builds), 1) self.assertEqual( len(tenant.semaphore_handler.semaphoreHolders("test-semaphore")), 1) # Start a second change in a different pipeline B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B') B.addApproval('Code-Review', 2) self.fake_gerrit.addEvent(B.addApproval('Approved', 1)) self.waitUntilSettled() # Still just the first change holds the lock self.assertEqual(len(self.builds), 1) self.assertEqual( len(tenant.semaphore_handler.semaphoreHolders("test-semaphore")), 1) # Now the second should run self.executor_server.hold_jobs_in_build = False self.executor_server.release() self.waitUntilSettled() self.assertHistory([ dict(name='check-job', result='SUCCESS', changes='1,1'), dict(name='gate-job', result='SUCCESS', changes='2,1'), ]) class TestSemaphoreMultiTenant(ZuulTestCase): tenant_config_file = 'config/multi-tenant-semaphore/main.yaml' def test_semaphore_tenant_isolation(self): "Test semaphores in multiple tenants" self.waitUntilSettled() tenant_one = self.scheds.first.sched.abide.tenants.get('tenant-one') tenant_two = self.scheds.first.sched.abide.tenants.get('tenant-two') self.executor_server.hold_jobs_in_build = True A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A') B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B') C = self.fake_gerrit.addFakeChange('org/project2', 'master', 'C') D = self.fake_gerrit.addFakeChange('org/project2', 'master', 'D') E = self.fake_gerrit.addFakeChange('org/project2', 'master', 'E') self.assertEqual( len(tenant_one.semaphore_handler.semaphoreHolders( "test-semaphore")), 0) self.assertEqual( len(tenant_two.semaphore_handler.semaphoreHolders( "test-semaphore")), 0) # add patches to project1 of tenant-one self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1)) self.waitUntilSettled() # one build of project1-test1 must run # semaphore of tenant-one must be acquired once # semaphore of tenant-two must not be acquired self.assertEqual(len(self.builds), 1) self.assertEqual(self.builds[0].name, 'project1-test1') self.assertEqual( len(tenant_one.semaphore_handler.semaphoreHolders( "test-semaphore")), 1) self.assertEqual( len(tenant_two.semaphore_handler.semaphoreHolders( "test-semaphore")), 0) # add patches to project2 of tenant-two self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1)) self.fake_gerrit.addEvent(D.getPatchsetCreatedEvent(1)) self.fake_gerrit.addEvent(E.getPatchsetCreatedEvent(1)) self.waitUntilSettled() # one build of project1-test1 must run # two builds of project2-test1 must run # semaphore of tenant-one must be acquired once # semaphore of tenant-two must be acquired twice self.assertEqual(len(self.builds), 3) self.assertEqual(self.builds[0].name, 'project1-test1') self.assertEqual(self.builds[1].name, 'project2-test1') self.assertEqual(self.builds[2].name, 'project2-test1') self.assertEqual( len(tenant_one.semaphore_handler.semaphoreHolders( "test-semaphore")), 1) self.assertEqual( len(tenant_two.semaphore_handler.semaphoreHolders( "test-semaphore")), 2) self.executor_server.release('project1-test1') self.waitUntilSettled() # one build of project1-test1 must run # two builds of project2-test1 must run # semaphore of tenant-one must be acquired once # semaphore of tenant-two must be acquired twice self.assertEqual(len(self.builds), 3) self.assertEqual(self.builds[0].name, 'project2-test1') self.assertEqual(self.builds[1].name, 'project2-test1') self.assertEqual(self.builds[2].name, 'project1-test1') self.assertEqual( len(tenant_one.semaphore_handler.semaphoreHolders( "test-semaphore")), 1) self.assertEqual( len(tenant_two.semaphore_handler.semaphoreHolders( "test-semaphore")), 2) self.executor_server.release('project2-test1') self.waitUntilSettled() # one build of project1-test1 must run # one build of project2-test1 must run # semaphore of tenant-one must be acquired once # semaphore of tenant-two must be acquired once self.assertEqual(len(self.builds), 2) self.assertEqual( len(tenant_one.semaphore_handler.semaphoreHolders( "test-semaphore")), 1) self.assertEqual( len(tenant_two.semaphore_handler.semaphoreHolders( "test-semaphore")), 1) self.executor_server.hold_jobs_in_build = False self.executor_server.release() self.waitUntilSettled() # no build must run # semaphore of tenant-one must not be acquired # semaphore of tenant-two must not be acquired self.assertEqual(len(self.builds), 0) self.assertEqual( len(tenant_one.semaphore_handler.semaphoreHolders( "test-semaphore")), 0) self.assertEqual( len(tenant_two.semaphore_handler.semaphoreHolders( "test-semaphore")), 0) self.assertEqual(A.reported, 1) self.assertEqual(B.reported, 1) class TestImplicitProject(ZuulTestCase): tenant_config_file = 'config/implicit-project/main.yaml' def test_implicit_project(self): # config project should work with implicit project name A = self.fake_gerrit.addFakeChange('common-config', 'master', 'A') self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) # untrusted project should work with implicit project name B = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1)) self.waitUntilSettled() self.assertEqual(A.data['status'], 'NEW') self.assertEqual(A.reported, 1) self.assertEqual(B.data['status'], 'NEW') self.assertEqual(B.reported, 1) self.assertHistory([ dict(name='test-common', result='SUCCESS', changes='1,1'), dict(name='test-common', result='SUCCESS', changes='2,1'), dict(name='test-project', result='SUCCESS', changes='2,1'), ], ordered=False) # now test adding a further project in repo in_repo_conf = textwrap.dedent( """ - job: name: test-project run: playbooks/test-project.yaml - job: name: test2-project run: playbooks/test-project.yaml - project: check: jobs: - test-project gate: jobs: - test-project - project: check: jobs: - test2-project gate: jobs: - test2-project """) file_dict = {'.zuul.yaml': in_repo_conf} C = self.fake_gerrit.addFakeChange('org/project', 'master', 'A', files=file_dict) C.addApproval('Code-Review', 2) self.fake_gerrit.addEvent(C.addApproval('Approved', 1)) self.waitUntilSettled() # change C must be merged self.assertEqual(C.data['status'], 'MERGED') self.assertEqual(C.reported, 2) self.assertHistory([ dict(name='test-common', result='SUCCESS', changes='1,1'), dict(name='test-common', result='SUCCESS', changes='2,1'), dict(name='test-project', result='SUCCESS', changes='2,1'), dict(name='test-common', result='SUCCESS', changes='3,1'), dict(name='test-project', result='SUCCESS', changes='3,1'), dict(name='test2-project', result='SUCCESS', changes='3,1'), ], ordered=False) class TestSemaphoreInRepo(ZuulTestCase): config_file = 'zuul-connections-gerrit-and-github.conf' tenant_config_file = 'config/in-repo/main.yaml' def test_semaphore_in_repo(self): "Test semaphores in repo config" # This tests dynamic semaphore handling in project repos. The semaphore # max value should not be evaluated dynamically but must be updated # after the change lands. self.waitUntilSettled() tenant = self.scheds.first.sched.abide.tenants.get('tenant-one') in_repo_conf = textwrap.dedent( """ - job: name: project-test1 - job: name: project-test2 run: playbooks/project-test2.yaml semaphore: test-semaphore - project: name: org/project tenant-one-gate: jobs: - project-test2 # the max value in dynamic layout must be ignored - semaphore: name: test-semaphore max: 2 """) in_repo_playbook = textwrap.dedent( """ - hosts: all tasks: [] """) file_dict = {'.zuul.yaml': in_repo_conf, 'playbooks/project-test2.yaml': in_repo_playbook} A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A', files=file_dict) B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B') C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C') B.setDependsOn(A, 1) C.setDependsOn(A, 1) self.executor_server.hold_jobs_in_build = True A.addApproval('Code-Review', 2) B.addApproval('Code-Review', 2) C.addApproval('Code-Review', 2) self.fake_gerrit.addEvent(A.addApproval('Approved', 1)) self.fake_gerrit.addEvent(B.addApproval('Approved', 1)) self.fake_gerrit.addEvent(C.addApproval('Approved', 1)) self.waitUntilSettled() # check that the layout in a queue item still has max value of 1 # for test-semaphore pipeline = tenant.layout.pipelines.get('tenant-one-gate') queue = None for queue_candidate in pipeline.queues: if queue_candidate.name == 'org/project': queue = queue_candidate break queue_item = queue.queue[0] item_dynamic_layout = pipeline.manager._layout_cache.get( queue_item.layout_uuid) self.assertIsNotNone(item_dynamic_layout) dynamic_test_semaphore = item_dynamic_layout.getSemaphore( self.scheds.first.sched.abide, 'test-semaphore') self.assertEqual(dynamic_test_semaphore.max, 1) # one build must be in queue, one semaphores acquired self.assertEqual(len(self.builds), 1) self.assertEqual(self.builds[0].name, 'project-test2') self.assertEqual( len(tenant.semaphore_handler.semaphoreHolders("test-semaphore")), 1) self.executor_server.release('project-test2') self.waitUntilSettled() # change A must be merged self.assertEqual(A.data['status'], 'MERGED') self.assertEqual(A.reported, 2) # send change-merged event as the gerrit mock doesn't send it self.fake_gerrit.addEvent(A.getChangeMergedEvent()) self.waitUntilSettled() # now that change A was merged, the new semaphore max must be effective tenant = self.scheds.first.sched.abide.tenants.get('tenant-one') self.assertEqual(tenant.layout.getSemaphore( self.scheds.first.sched.abide, 'test-semaphore').max, 2) # two builds must be in queue, two semaphores acquired self.assertEqual(len(self.builds), 2) self.assertEqual(self.builds[0].name, 'project-test2') self.assertEqual(self.builds[1].name, 'project-test2') self.assertEqual( len(tenant.semaphore_handler.semaphoreHolders("test-semaphore")), 2) self.executor_server.release('project-test2') self.waitUntilSettled() self.assertEqual(len(self.builds), 0) self.assertEqual( len(tenant.semaphore_handler.semaphoreHolders("test-semaphore")), 0 ) self.executor_server.hold_jobs_in_build = False self.executor_server.release() self.waitUntilSettled() self.assertEqual(len(self.builds), 0) self.assertEqual(A.reported, 2) self.assertEqual(B.reported, 2) self.assertEqual(C.reported, 2) class TestSchedulerBranchMatcher(ZuulTestCase): @simple_layout('layouts/matcher-test.yaml') def test_job_branch_ignored(self): ''' Test that branch matching logic works. The 'ignore-branch' job has a branch matcher that is supposed to match every branch except for the 'featureA' branch, so it should not be run on a change to that branch. ''' self.create_branch('org/project', 'featureA') self.fake_gerrit.addEvent( self.fake_gerrit.getFakeBranchCreatedEvent( 'org/project', 'featureA')) self.waitUntilSettled() A = self.fake_gerrit.addFakeChange('org/project', 'featureA', 'A') A.addApproval('Code-Review', 2) self.fake_gerrit.addEvent(A.addApproval('Approved', 1)) self.waitUntilSettled() self.printHistory() self.assertEqual(self.getJobFromHistory('project-test1').result, 'SUCCESS') self.assertJobNotInHistory('ignore-branch') self.assertEqual(A.data['status'], 'MERGED') self.assertEqual(A.reported, 2, "A should report start and success") self.assertIn('gate', A.messages[1], "A should transit gate") class TestSchedulerFailFast(ZuulTestCase): tenant_config_file = 'config/fail-fast/main.yaml' def test_fail_fast(self): """ Tests that a pipeline that is flagged with fail-fast aborts jobs early. """ self.executor_server.hold_jobs_in_build = True self.fake_nodepool.pause() A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') self.executor_server.failJob('project-test1', A) self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) self.waitUntilSettled() self.waitUntilSettled() self.assertEqual(len(self.builds), 1) self.assertEqual(self.builds[0].name, 'project-merge') self.executor_server.release('project-merge') self.waitUntilSettled() # Now project-test1, project-test2 and project-test6 # should be running self.assertEqual(len(self.builds), 3) # Release project-test1 which will fail self.executor_server.release('project-test1') self.waitUntilSettled() self.fake_nodepool.unpause() self.waitUntilSettled() # Now project-test2 must be aborted self.assertEqual(len(self.builds), 0) self.assertEqual(A.reported, 1) self.assertHistory([ dict(name='project-merge', result='SUCCESS', changes='1,1'), dict(name='project-test1', result='FAILURE', changes='1,1'), dict(name='project-test2', result='ABORTED', changes='1,1'), dict(name='project-test6', result='ABORTED', changes='1,1'), ], ordered=False) def test_fail_fast_gate(self): """ Tests that a pipeline that is flagged with fail-fast aborts jobs early. """ self.executor_server.hold_jobs_in_build = True A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') A.addApproval('Code-Review', 2) self.fake_gerrit.addEvent(A.addApproval('Approved', 1)) self.waitUntilSettled() B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B') self.executor_server.failJob('project-test1', B) B.addApproval('Code-Review', 2) self.fake_gerrit.addEvent(B.addApproval('Approved', 1)) self.waitUntilSettled() self.assertEqual(len(self.builds), 4) # Release project-test1 which will fail self.builds[2].release() self.waitUntilSettled() # We should only have the builds from change A now self.assertEqual(len(self.builds), 2) self.assertEqual(self.builds[0].name, 'project-test1') self.assertEqual(self.builds[1].name, 'project-test2') # But both changes should still be in the pipeline tenant = self.scheds.first.sched.abide.tenants.get('tenant-one') items = tenant.layout.pipelines['gate'].getAllItems() self.assertEqual(len(items), 2) self.assertEqual(A.reported, 1) self.assertEqual(B.reported, 1) # Release change A self.executor_server.hold_jobs_in_build = False self.executor_server.release() self.waitUntilSettled() self.assertEqual(len(self.builds), 0) self.assertEqual(A.reported, 2) self.assertEqual(B.reported, 2) self.assertHistory([ dict(name='project-test1', result='SUCCESS', changes='1,1'), dict(name='project-test2', result='SUCCESS', changes='1,1'), dict(name='project-test1', result='FAILURE', changes='1,1 2,1'), dict(name='project-test2', result='ABORTED', changes='1,1 2,1'), ], ordered=False) def test_fail_fast_nonvoting(self): """ Tests that a pipeline that is flagged with fail-fast doesn't abort jobs due to a non-voting job. """ self.executor_server.hold_jobs_in_build = True A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') self.executor_server.failJob('project-test6', A) self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) self.waitUntilSettled() self.waitUntilSettled() self.assertEqual(len(self.builds), 2) self.assertEqual(self.builds[0].name, 'project-merge') self.executor_server.release('project-merge') self.waitUntilSettled() # Now project-test1, project-test2, project-test5 and project-test6 # should be running self.assertEqual(len(self.builds), 4) # Release project-test6 which will fail self.executor_server.release('project-test6') self.waitUntilSettled() # Now project-test1, project-test2 and project-test5 should be running self.assertEqual(len(self.builds), 3) self.executor_server.hold_jobs_in_build = False self.executor_server.release() self.waitUntilSettled() self.assertEqual(len(self.builds), 0) self.assertEqual(A.reported, 1) self.assertHistory([ dict(name='project-merge', result='SUCCESS', changes='1,1'), dict(name='project-test1', result='SUCCESS', changes='1,1'), dict(name='project-test2', result='SUCCESS', changes='1,1'), dict(name='project-test3', result='SUCCESS', changes='1,1'), dict(name='project-test4', result='SUCCESS', changes='1,1'), dict(name='project-test5', result='SUCCESS', changes='1,1'), dict(name='project-test6', result='FAILURE', changes='1,1'), ], ordered=False) def test_fail_fast_retry(self): """ Tests that a retried build doesn't trigger fail-fast. """ self.executor_server.hold_jobs_in_build = True A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) self.waitUntilSettled() # project-merge and project-test5 self.assertEqual(len(self.builds), 2) # Force a retry of first build self.builds[0].requeue = True self.executor_server.hold_jobs_in_build = False self.executor_server.release() self.waitUntilSettled() self.assertEqual(len(self.builds), 0) self.assertEqual(A.reported, 1) self.assertHistory([ dict(name='project-merge', result=None, changes='1,1'), dict(name='project-merge', result='SUCCESS', changes='1,1'), dict(name='project-test1', result='SUCCESS', changes='1,1'), dict(name='project-test2', result='SUCCESS', changes='1,1'), dict(name='project-test3', result='SUCCESS', changes='1,1'), dict(name='project-test4', result='SUCCESS', changes='1,1'), dict(name='project-test5', result='SUCCESS', changes='1,1'), dict(name='project-test6', result='SUCCESS', changes='1,1'), ], ordered=False) class TestPipelineSupersedes(ZuulTestCase): @simple_layout('layouts/pipeline-supercedes.yaml') def test_supercedes(self): """ Tests that a pipeline that is flagged with fail-fast aborts jobs early. """ self.executor_server.hold_jobs_in_build = True A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) self.waitUntilSettled() self.assertEqual(len(self.builds), 1) self.assertEqual(self.builds[0].name, 'test-job') A.addApproval('Code-Review', 2) self.fake_gerrit.addEvent(A.addApproval('Approved', 1)) self.waitUntilSettled() self.assertEqual(len(self.builds), 1) self.assertEqual(self.builds[0].name, 'test-job') self.assertEqual(self.builds[0].pipeline, 'gate') self.executor_server.hold_jobs_in_build = False self.executor_server.release() self.waitUntilSettled() self.assertEqual(len(self.builds), 0) self.assertEqual(A.reported, 2) self.assertEqual(A.data['status'], 'MERGED') self.assertHistory([ dict(name='test-job', result='ABORTED', changes='1,1'), dict(name='test-job', result='SUCCESS', changes='1,1'), ], ordered=False) class TestSchedulerExcludeAll(ZuulTestCase): tenant_config_file = 'config/two-tenant/exclude-all.yaml' def test_skip_reconfig_exclude_all(self): """Test that we don't trigger a reconfiguration for a tenant when the changed project excludes all config.""" config = textwrap.dedent( """ - job: name: project2-test parent: test - project: check: jobs: - project2-test """) file_dict = {'zuul.yaml': config} A = self.fake_gerrit.addFakeChange('org/project2', 'master', 'A', files=file_dict) self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) self.waitUntilSettled() self.assertHistory([ dict(name='project2-test', result='SUCCESS', changes='1,1'), ]) sched = self.scheds.first.sched tenant_one_layout_state = sched.local_layout_state["tenant-one"] tenant_two_layout_state = sched.local_layout_state["tenant-two"] A.setMerged() self.fake_gerrit.addEvent(A.getChangeMergedEvent()) self.waitUntilSettled() # We don't expect a reconfiguration for tenant-one as it excludes # all config of org/project2. self.assertEqual(sched.local_layout_state["tenant-one"], tenant_one_layout_state) # As tenant-two includes the config from org/project2, the merge of # change A should have triggered a reconfig. self.assertGreater(sched.local_layout_state["tenant-two"], tenant_two_layout_state) B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B') self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1)) self.waitUntilSettled() self.assertHistory([ dict(name='project2-test', result='SUCCESS', changes='1,1'), dict(name='project2-test', result='SUCCESS', changes='2,1'), ]) class TestReportBuildPage(ZuulTestCase): tenant_config_file = 'config/build-page/main.yaml' def test_tenant_url(self): """ Test that the tenant url is used in reporting the build page. """ A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A') self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) self.waitUntilSettled() self.assertHistory([ dict(name='python27', result='SUCCESS', changes='1,1'), ]) self.assertIn('python27 https://one.example.com/build/', A.messages[0]) def test_base_url(self): """ Test that the web base url is used in reporting the build page. """ A = self.fake_gerrit.addFakeChange('org/project2', 'master', 'A') self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) self.waitUntilSettled() self.assertHistory([ dict(name='python27', result='SUCCESS', changes='1,1'), ]) self.assertIn('python27 https://zuul.example.com/t/tenant-two/build/', A.messages[0]) def test_no_build_page(self): """ Test that we fall back to the old behavior if the tenant is not configured to report the build page """ A = self.fake_gerrit.addFakeChange('org/project3', 'master', 'A') self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) self.waitUntilSettled() self.assertHistory([ dict(name='python27', result='SUCCESS', changes='1,1'), ]) self.assertIn('python27 https://', A.messages[0]) class TestSchedulerSmartReconfiguration(ZuulTestCase): tenant_config_file = 'config/multi-tenant/main.yaml' def _test_smart_reconfiguration(self, command_socket=False): """ Tests that smart reconfiguration works In this scenario we have the tenants tenant-one, tenant-two and tenant-three. We make the following changes and then trigger a smart reconfiguration: - tenant-one remains unchanged - tenant-two gets another repo - tenant-three gets removed completely - tenant-four is a new tenant """ self.executor_server.hold_jobs_in_build = True # Create changes for all tenants A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A') self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B') self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1)) C = self.fake_gerrit.addFakeChange('org/project3', 'master', 'C') self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1)) # record previous tenant reconfiguration time, which may not be set old_one = self.scheds.first.sched.tenant_layout_state.get( 'tenant-one', EMPTY_LAYOUT_STATE) old_two = self.scheds.first.sched.tenant_layout_state.get( 'tenant-two', EMPTY_LAYOUT_STATE) self.waitUntilSettled() self.newTenantConfig('config/multi-tenant/main-reconfig.yaml') del self.merge_job_history self.scheds.execute( lambda app: app.smartReconfigure(command_socket=command_socket)) # Wait for smart reconfiguration. Only tenant-two should be # reconfigured. Note that waitUntilSettled is not # reliable here because the reconfigure event may arrive in the # event queue after waitUntilSettled. start = time.time() while True: if time.time() - start > 30: raise Exception("Timeout waiting for smart reconfiguration") new_two = self.scheds.first.sched.tenant_layout_state.get( 'tenant-two', EMPTY_LAYOUT_STATE) if old_two < new_two: break else: time.sleep(0.1) self.waitUntilSettled() # We're only adding two new repos, so we should only need to # issue 2 cat jobs. cat_jobs = self.merge_job_history.get(zuul.model.MergeRequest.CAT) self.assertEqual(len(cat_jobs), 2) # Ensure that tenant-one has not been reconfigured new_one = self.scheds.first.sched.tenant_layout_state.get( 'tenant-one', EMPTY_LAYOUT_STATE) self.assertEqual(old_one, new_one) self.executor_server.hold_jobs_in_build = False self.executor_server.release() self.waitUntilSettled() # Changes in tenant-one and tenant-two have to be reported self.assertEqual(1, A.reported) self.assertEqual(1, B.reported) # The tenant-three has been removed so nothing should be reported self.assertEqual(0, C.reported) self.assertNotIn('tenant-three', self.scheds.first.sched.abide.tenants) # Verify known tenants expected_tenants = {'tenant-one', 'tenant-two', 'tenant-four'} self.assertEqual(expected_tenants, self.scheds.first.sched.abide.tenants.keys()) self.assertIsNotNone( self.scheds.first.sched.tenant_layout_state.get('tenant-four'), 'Tenant tenant-four should exist now.' ) # Test that the new tenant-four actually works D = self.fake_gerrit.addFakeChange('org/project4', 'master', 'D') self.fake_gerrit.addEvent(D.getPatchsetCreatedEvent(1)) self.waitUntilSettled() self.assertEqual(1, D.reported) # Test that the new project in tenant-two works B2 = self.fake_gerrit.addFakeChange('org/project2b', 'master', 'B2') self.fake_gerrit.addEvent(B2.getPatchsetCreatedEvent(1)) self.waitUntilSettled() self.assertEqual(1, B2.reported) def test_smart_reconfiguration(self): "Test that live reconfiguration works" self._test_smart_reconfiguration() def test_smart_reconfiguration_command_socket(self): "Test that live reconfiguration works using command socket" self._test_smart_reconfiguration(command_socket=True) class TestReconfigureBranch(ZuulTestCase): def _setupTenantReconfigureTime(self): self.old = self.scheds.first.sched.tenant_layout_state.get( 'tenant-one', EMPTY_LAYOUT_STATE) def _createBranch(self): self.create_branch('org/project1', 'stable') self.fake_gerrit.addEvent( self.fake_gerrit.getFakeBranchCreatedEvent( 'org/project1', 'stable')) self.waitUntilSettled() def _deleteBranch(self): self.delete_branch('org/project1', 'stable') self.fake_gerrit.addEvent( self.fake_gerrit.getFakeBranchDeletedEvent( 'org/project1', 'stable')) self.waitUntilSettled() def _expectReconfigure(self, doReconfigure): new = self.scheds.first.sched.tenant_layout_state.get( 'tenant-one', EMPTY_LAYOUT_STATE) if doReconfigure: self.assertLess(self.old, new) else: self.assertEqual(self.old, new) self.old = new class TestReconfigureBranchCreateDeleteSshHttp(TestReconfigureBranch): tenant_config_file = 'config/single-tenant/main.yaml' config_file = 'zuul-gerrit-web.conf' def test_reconfigure_cache_branch_create_delete(self): "Test that cache is updated clear on branch creation/deletion" self._setupTenantReconfigureTime() self._createBranch() self._expectReconfigure(True) self._deleteBranch() self._expectReconfigure(True) class TestReconfigureBranchCreateDeleteSsh(TestReconfigureBranch): tenant_config_file = 'config/single-tenant/main.yaml' def test_reconfigure_cache_branch_create_delete(self): "Test that cache is updated clear on branch creation/deletion" self._setupTenantReconfigureTime() self._createBranch() self._expectReconfigure(True) self._deleteBranch() self._expectReconfigure(True) class TestReconfigureBranchCreateDeleteHttp(TestReconfigureBranch): tenant_config_file = 'config/single-tenant/main.yaml' config_file = 'zuul-gerrit-no-stream.conf' def test_reconfigure_cache_branch_create_delete(self): "Test that cache is updated clear on branch creation/deletion" self._setupTenantReconfigureTime() self._createBranch() self._expectReconfigure(True) self._deleteBranch() self._expectReconfigure(True) class TestEventProcessing(ZuulTestCase): tenant_config_file = 'config/event-processing/main.yaml' # Some regression tests for ZK-distributed event processing def test_independent_tenants(self): # Test that an exception in one tenant doesn't break others orig = zuul.scheduler.Scheduler._forward_trigger_event def patched_forward(obj, *args, **kw): if args[1].name == 'tenant-one': raise Exception("test") return orig(obj, *args, **kw) self.useFixture(fixtures.MonkeyPatch( 'zuul.scheduler.Scheduler._forward_trigger_event', patched_forward)) A = self.fake_gerrit.addFakeChange('org/project2', 'master', 'A') self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) self.waitUntilSettled() self.assertHistory([ dict(name='checkjob', result='SUCCESS', changes='1,1'), ], ordered=False) @skip("Failure can only be detected in logs; see test_ref_equality") def test_change_types(self): # Test that when we decide whether to forward events, we can # compare items with different change types (branch vs # change). # We can't detect a failure here except by observing the logs; # this test is left in case that's useful in the future, but # automated detection of this failure case is handled by # test_ref_equality. # Enqueue a tag self.executor_server.hold_jobs_in_build = True event = self.fake_gerrit.addFakeTag('org/project1', 'master', 'foo') self.fake_gerrit.addEvent(event) # Enqueue a change and make sure the scheduler is able to # compare the two when forwarding the event A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A') self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) self.waitUntilSettled() self.executor_server.hold_jobs_in_build = False self.executor_server.release() self.waitUntilSettled() self.assertHistory([ dict(name='tagjob', result='SUCCESS'), dict(name='checkjob', result='SUCCESS', changes='1,1'), ], ordered=False) class TestWaitForInit(ZuulTestCase): tenant_config_file = 'config/single-tenant/main.yaml' wait_for_init = True def setUp(self): with self.assertLogs('zuul.Scheduler-0', level='DEBUG') as full_logs: super().setUp() self.assertRegexInList('Waiting for tenant initialization', full_logs.output) def test_wait_for_init(self): A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') A.addApproval('Code-Review', 2) self.fake_gerrit.addEvent(A.addApproval('Approved', 1)) self.waitUntilSettled()