#!/usr/bin/env python # Copyright 2012 Hewlett-Packard Development Company, L.P. # Copyright 2014 Wikimedia Foundation Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. import logging import os import shutil import time import git import zuul.lib.cloner from tests.base import ZuulTestCase class TestCloner(ZuulTestCase): log = logging.getLogger("zuul.test.cloner") workspace_root = None def setUp(self): self.skip("Disabled for early v3 development") super(TestCloner, self).setUp() self.workspace_root = os.path.join(self.test_root, 'workspace') self.updateConfigLayout( 'tests/fixtures/layout-cloner.yaml') self.sched.reconfigure(self.config) self.registerJobs() def getWorkspaceRepos(self, projects): repos = {} for project in projects: repos[project] = git.Repo( os.path.join(self.workspace_root, project)) return repos def getUpstreamRepos(self, projects): repos = {} for project in projects: repos[project] = git.Repo( os.path.join(self.upstream_root, project)) return repos def test_cache_dir(self): projects = ['org/project1', 'org/project2'] cache_root = os.path.join(self.test_root, "cache") for project in projects: upstream_repo_path = os.path.join(self.upstream_root, project) cache_repo_path = os.path.join(cache_root, project) git.Repo.clone_from(upstream_repo_path, cache_repo_path) self.worker.hold_jobs_in_build = True A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A') A.addApproval('CRVW', 2) self.fake_gerrit.addEvent(A.addApproval('APRV', 1)) self.waitUntilSettled() self.assertEquals(1, len(self.builds), "One build is running") B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B') B.setMerged() upstream = self.getUpstreamRepos(projects) states = [{ 'org/project1': self.builds[0].parameters['ZUUL_COMMIT'], 'org/project2': str(upstream['org/project2'].commit('master')), }] for number, build in enumerate(self.builds): self.log.debug("Build parameters: %s", build.parameters) cloner = zuul.lib.cloner.Cloner( git_base_url=self.upstream_root, projects=projects, workspace=self.workspace_root, zuul_branch=build.parameters['ZUUL_BRANCH'], zuul_ref=build.parameters['ZUUL_REF'], zuul_url=self.git_root, cache_dir=cache_root, ) cloner.execute() work = self.getWorkspaceRepos(projects) state = states[number] for project in projects: self.assertEquals(state[project], str(work[project].commit('HEAD')), 'Project %s commit for build %s should ' 'be correct' % (project, number)) work = self.getWorkspaceRepos(projects) upstream_repo_path = os.path.join(self.upstream_root, 'org/project1') self.assertEquals( work['org/project1'].remotes.origin.url, upstream_repo_path, 'workspace repo origin should be upstream, not cache' ) self.worker.hold_jobs_in_build = False self.worker.release() self.waitUntilSettled() def test_one_branch(self): self.worker.hold_jobs_in_build = True projects = ['org/project1', 'org/project2'] A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A') B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B') A.addApproval('CRVW', 2) B.addApproval('CRVW', 2) self.fake_gerrit.addEvent(A.addApproval('APRV', 1)) self.fake_gerrit.addEvent(B.addApproval('APRV', 1)) self.waitUntilSettled() self.assertEquals(2, len(self.builds), "Two builds are running") upstream = self.getUpstreamRepos(projects) states = [ {'org/project1': self.builds[0].parameters['ZUUL_COMMIT'], 'org/project2': str(upstream['org/project2'].commit('master')), }, {'org/project1': self.builds[0].parameters['ZUUL_COMMIT'], 'org/project2': self.builds[1].parameters['ZUUL_COMMIT'], }, ] for number, build in enumerate(self.builds): self.log.debug("Build parameters: %s", build.parameters) cloner = zuul.lib.cloner.Cloner( git_base_url=self.upstream_root, projects=projects, workspace=self.workspace_root, zuul_branch=build.parameters['ZUUL_BRANCH'], zuul_ref=build.parameters['ZUUL_REF'], zuul_url=self.git_root, ) cloner.execute() work = self.getWorkspaceRepos(projects) state = states[number] for project in projects: self.assertEquals(state[project], str(work[project].commit('HEAD')), 'Project %s commit for build %s should ' 'be correct' % (project, number)) shutil.rmtree(self.workspace_root) self.worker.hold_jobs_in_build = False self.worker.release() self.waitUntilSettled() def test_multi_branch(self): self.worker.hold_jobs_in_build = True projects = ['org/project1', 'org/project2', 'org/project3', 'org/project4'] self.create_branch('org/project2', 'stable/havana') self.create_branch('org/project4', 'stable/havana') A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A') B = self.fake_gerrit.addFakeChange('org/project2', 'stable/havana', 'B') C = self.fake_gerrit.addFakeChange('org/project3', 'master', 'C') A.addApproval('CRVW', 2) B.addApproval('CRVW', 2) C.addApproval('CRVW', 2) self.fake_gerrit.addEvent(A.addApproval('APRV', 1)) self.fake_gerrit.addEvent(B.addApproval('APRV', 1)) self.fake_gerrit.addEvent(C.addApproval('APRV', 1)) self.waitUntilSettled() self.assertEquals(3, len(self.builds), "Three builds are running") upstream = self.getUpstreamRepos(projects) states = [ {'org/project1': self.builds[0].parameters['ZUUL_COMMIT'], 'org/project2': str(upstream['org/project2'].commit('master')), 'org/project3': str(upstream['org/project3'].commit('master')), 'org/project4': str(upstream['org/project4']. commit('master')), }, {'org/project1': self.builds[0].parameters['ZUUL_COMMIT'], 'org/project2': self.builds[1].parameters['ZUUL_COMMIT'], 'org/project3': str(upstream['org/project3'].commit('master')), 'org/project4': str(upstream['org/project4']. commit('stable/havana')), }, {'org/project1': self.builds[0].parameters['ZUUL_COMMIT'], 'org/project2': str(upstream['org/project2'].commit('master')), 'org/project3': self.builds[2].parameters['ZUUL_COMMIT'], 'org/project4': str(upstream['org/project4']. commit('master')), }, ] for number, build in enumerate(self.builds): self.log.debug("Build parameters: %s", build.parameters) cloner = zuul.lib.cloner.Cloner( git_base_url=self.upstream_root, projects=projects, workspace=self.workspace_root, zuul_branch=build.parameters['ZUUL_BRANCH'], zuul_ref=build.parameters['ZUUL_REF'], zuul_url=self.git_root, ) cloner.execute() work = self.getWorkspaceRepos(projects) state = states[number] for project in projects: self.assertEquals(state[project], str(work[project].commit('HEAD')), 'Project %s commit for build %s should ' 'be correct' % (project, number)) shutil.rmtree(self.workspace_root) self.worker.hold_jobs_in_build = False self.worker.release() self.waitUntilSettled() def test_upgrade(self): # Simulates an upgrade test self.worker.hold_jobs_in_build = True projects = ['org/project1', 'org/project2', 'org/project3', 'org/project4', 'org/project5', 'org/project6'] self.create_branch('org/project2', 'stable/havana') self.create_branch('org/project3', 'stable/havana') self.create_branch('org/project4', 'stable/havana') self.create_branch('org/project5', 'stable/havana') A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A') B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B') C = self.fake_gerrit.addFakeChange('org/project3', 'stable/havana', 'C') D = self.fake_gerrit.addFakeChange('org/project3', 'master', 'D') E = self.fake_gerrit.addFakeChange('org/project4', 'stable/havana', 'E') A.addApproval('CRVW', 2) B.addApproval('CRVW', 2) C.addApproval('CRVW', 2) D.addApproval('CRVW', 2) E.addApproval('CRVW', 2) self.fake_gerrit.addEvent(A.addApproval('APRV', 1)) self.fake_gerrit.addEvent(B.addApproval('APRV', 1)) self.fake_gerrit.addEvent(C.addApproval('APRV', 1)) self.fake_gerrit.addEvent(D.addApproval('APRV', 1)) self.fake_gerrit.addEvent(E.addApproval('APRV', 1)) self.waitUntilSettled() self.assertEquals(5, len(self.builds), "Five builds are running") # Check the old side of the upgrade first upstream = self.getUpstreamRepos(projects) states = [ {'org/project1': self.builds[0].parameters['ZUUL_COMMIT'], 'org/project2': str(upstream['org/project2'].commit( 'stable/havana')), 'org/project3': str(upstream['org/project3'].commit( 'stable/havana')), 'org/project4': str(upstream['org/project4'].commit( 'stable/havana')), 'org/project5': str(upstream['org/project5'].commit( 'stable/havana')), 'org/project6': str(upstream['org/project6'].commit('master')), }, {'org/project1': self.builds[0].parameters['ZUUL_COMMIT'], 'org/project2': str(upstream['org/project2'].commit( 'stable/havana')), 'org/project3': str(upstream['org/project3'].commit( 'stable/havana')), 'org/project4': str(upstream['org/project4'].commit( 'stable/havana')), 'org/project5': str(upstream['org/project5'].commit( 'stable/havana')), 'org/project6': str(upstream['org/project6'].commit('master')), }, {'org/project1': self.builds[0].parameters['ZUUL_COMMIT'], 'org/project2': str(upstream['org/project2'].commit( 'stable/havana')), 'org/project3': self.builds[2].parameters['ZUUL_COMMIT'], 'org/project4': str(upstream['org/project4'].commit( 'stable/havana')), 'org/project5': str(upstream['org/project5'].commit( 'stable/havana')), 'org/project6': str(upstream['org/project6'].commit('master')), }, {'org/project1': self.builds[0].parameters['ZUUL_COMMIT'], 'org/project2': str(upstream['org/project2'].commit( 'stable/havana')), 'org/project3': self.builds[2].parameters['ZUUL_COMMIT'], 'org/project4': str(upstream['org/project4'].commit( 'stable/havana')), 'org/project5': str(upstream['org/project5'].commit( 'stable/havana')), 'org/project6': str(upstream['org/project6'].commit('master')), }, {'org/project1': self.builds[0].parameters['ZUUL_COMMIT'], 'org/project2': str(upstream['org/project2'].commit( 'stable/havana')), 'org/project3': self.builds[2].parameters['ZUUL_COMMIT'], 'org/project4': self.builds[4].parameters['ZUUL_COMMIT'], 'org/project5': str(upstream['org/project5'].commit( 'stable/havana')), 'org/project6': str(upstream['org/project6'].commit('master')), }, ] for number, build in enumerate(self.builds): self.log.debug("Build parameters: %s", build.parameters) cloner = zuul.lib.cloner.Cloner( git_base_url=self.upstream_root, projects=projects, workspace=self.workspace_root, zuul_branch=build.parameters['ZUUL_BRANCH'], zuul_ref=build.parameters['ZUUL_REF'], zuul_url=self.git_root, branch='stable/havana', # Old branch for upgrade ) cloner.execute() work = self.getWorkspaceRepos(projects) state = states[number] for project in projects: self.assertEquals(state[project], str(work[project].commit('HEAD')), 'Project %s commit for build %s should ' 'be correct on old side of upgrade' % (project, number)) shutil.rmtree(self.workspace_root) # Check the new side of the upgrade states = [ {'org/project1': self.builds[0].parameters['ZUUL_COMMIT'], 'org/project2': str(upstream['org/project2'].commit('master')), 'org/project3': str(upstream['org/project3'].commit('master')), 'org/project4': str(upstream['org/project4'].commit('master')), 'org/project5': str(upstream['org/project5'].commit('master')), 'org/project6': str(upstream['org/project6'].commit('master')), }, {'org/project1': self.builds[0].parameters['ZUUL_COMMIT'], 'org/project2': self.builds[1].parameters['ZUUL_COMMIT'], 'org/project3': str(upstream['org/project3'].commit('master')), 'org/project4': str(upstream['org/project4'].commit('master')), 'org/project5': str(upstream['org/project5'].commit('master')), 'org/project6': str(upstream['org/project6'].commit('master')), }, {'org/project1': self.builds[0].parameters['ZUUL_COMMIT'], 'org/project2': self.builds[1].parameters['ZUUL_COMMIT'], 'org/project3': str(upstream['org/project3'].commit('master')), 'org/project4': str(upstream['org/project4'].commit('master')), 'org/project5': str(upstream['org/project5'].commit('master')), 'org/project6': str(upstream['org/project6'].commit('master')), }, {'org/project1': self.builds[0].parameters['ZUUL_COMMIT'], 'org/project2': self.builds[1].parameters['ZUUL_COMMIT'], 'org/project3': self.builds[3].parameters['ZUUL_COMMIT'], 'org/project4': str(upstream['org/project4'].commit('master')), 'org/project5': str(upstream['org/project5'].commit('master')), 'org/project6': str(upstream['org/project6'].commit('master')), }, {'org/project1': self.builds[0].parameters['ZUUL_COMMIT'], 'org/project2': self.builds[1].parameters['ZUUL_COMMIT'], 'org/project3': self.builds[3].parameters['ZUUL_COMMIT'], 'org/project4': str(upstream['org/project4'].commit('master')), 'org/project5': str(upstream['org/project5'].commit('master')), 'org/project6': str(upstream['org/project6'].commit('master')), }, ] for number, build in enumerate(self.builds): self.log.debug("Build parameters: %s", build.parameters) cloner = zuul.lib.cloner.Cloner( git_base_url=self.upstream_root, projects=projects, workspace=self.workspace_root, zuul_branch=build.parameters['ZUUL_BRANCH'], zuul_ref=build.parameters['ZUUL_REF'], zuul_url=self.git_root, branch='master', # New branch for upgrade ) cloner.execute() work = self.getWorkspaceRepos(projects) state = states[number] for project in projects: self.assertEquals(state[project], str(work[project].commit('HEAD')), 'Project %s commit for build %s should ' 'be correct on old side of upgrade' % (project, number)) shutil.rmtree(self.workspace_root) self.worker.hold_jobs_in_build = False self.worker.release() self.waitUntilSettled() def test_project_override(self): self.worker.hold_jobs_in_build = True projects = ['org/project1', 'org/project2', 'org/project3', 'org/project4', 'org/project5', 'org/project6'] self.create_branch('org/project3', 'stable/havana') self.create_branch('org/project4', 'stable/havana') self.create_branch('org/project6', 'stable/havana') A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A') B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B') C = self.fake_gerrit.addFakeChange('org/project2', 'master', 'C') D = self.fake_gerrit.addFakeChange('org/project3', 'stable/havana', 'D') A.addApproval('CRVW', 2) B.addApproval('CRVW', 2) C.addApproval('CRVW', 2) D.addApproval('CRVW', 2) self.fake_gerrit.addEvent(A.addApproval('APRV', 1)) self.fake_gerrit.addEvent(B.addApproval('APRV', 1)) self.fake_gerrit.addEvent(C.addApproval('APRV', 1)) self.fake_gerrit.addEvent(D.addApproval('APRV', 1)) self.waitUntilSettled() self.assertEquals(4, len(self.builds), "Four builds are running") upstream = self.getUpstreamRepos(projects) states = [ {'org/project1': self.builds[0].parameters['ZUUL_COMMIT'], 'org/project2': str(upstream['org/project2'].commit('master')), 'org/project3': str(upstream['org/project3'].commit('master')), 'org/project4': str(upstream['org/project4'].commit('master')), 'org/project5': str(upstream['org/project5'].commit('master')), 'org/project6': str(upstream['org/project6'].commit('master')), }, {'org/project1': self.builds[1].parameters['ZUUL_COMMIT'], 'org/project2': str(upstream['org/project2'].commit('master')), 'org/project3': str(upstream['org/project3'].commit('master')), 'org/project4': str(upstream['org/project4'].commit('master')), 'org/project5': str(upstream['org/project5'].commit('master')), 'org/project6': str(upstream['org/project6'].commit('master')), }, {'org/project1': self.builds[1].parameters['ZUUL_COMMIT'], 'org/project2': self.builds[2].parameters['ZUUL_COMMIT'], 'org/project3': str(upstream['org/project3'].commit('master')), 'org/project4': str(upstream['org/project4'].commit('master')), 'org/project5': str(upstream['org/project5'].commit('master')), 'org/project6': str(upstream['org/project6'].commit('master')), }, {'org/project1': self.builds[1].parameters['ZUUL_COMMIT'], 'org/project2': self.builds[2].parameters['ZUUL_COMMIT'], 'org/project3': self.builds[3].parameters['ZUUL_COMMIT'], 'org/project4': str(upstream['org/project4'].commit('master')), 'org/project5': str(upstream['org/project5'].commit('master')), 'org/project6': str(upstream['org/project6'].commit( 'stable/havana')), }, ] for number, build in enumerate(self.builds): self.log.debug("Build parameters: %s", build.parameters) cloner = zuul.lib.cloner.Cloner( git_base_url=self.upstream_root, projects=projects, workspace=self.workspace_root, zuul_branch=build.parameters['ZUUL_BRANCH'], zuul_ref=build.parameters['ZUUL_REF'], zuul_url=self.git_root, project_branches={'org/project4': 'master'}, ) cloner.execute() work = self.getWorkspaceRepos(projects) state = states[number] for project in projects: self.assertEquals(state[project], str(work[project].commit('HEAD')), 'Project %s commit for build %s should ' 'be correct' % (project, number)) shutil.rmtree(self.workspace_root) self.worker.hold_jobs_in_build = False self.worker.release() self.waitUntilSettled() def test_periodic(self): self.worker.hold_jobs_in_build = True self.create_branch('org/project', 'stable/havana') self.updateConfigLayout( 'tests/fixtures/layout-timer.yaml') self.sched.reconfigure(self.config) self.registerJobs() # The pipeline triggers every second, so we should have seen # several by now. time.sleep(5) self.waitUntilSettled() builds = self.builds[:] self.worker.hold_jobs_in_build = False # Stop queuing timer triggered jobs so that the assertions # below don't race against more jobs being queued. self.updateConfigLayout( 'tests/fixtures/layout-no-timer.yaml') self.sched.reconfigure(self.config) self.registerJobs() self.worker.release() self.waitUntilSettled() projects = ['org/project'] self.assertEquals(2, len(builds), "Two builds are running") upstream = self.getUpstreamRepos(projects) states = [ {'org/project': str(upstream['org/project'].commit('stable/havana')), }, {'org/project': str(upstream['org/project'].commit('stable/havana')), }, ] for number, build in enumerate(builds): self.log.debug("Build parameters: %s", build.parameters) cloner = zuul.lib.cloner.Cloner( git_base_url=self.upstream_root, projects=projects, workspace=self.workspace_root, zuul_branch=build.parameters.get('ZUUL_BRANCH', None), zuul_ref=build.parameters.get('ZUUL_REF', None), zuul_url=self.git_root, branch='stable/havana', ) cloner.execute() work = self.getWorkspaceRepos(projects) state = states[number] for project in projects: self.assertEquals(state[project], str(work[project].commit('HEAD')), 'Project %s commit for build %s should ' 'be correct' % (project, number)) shutil.rmtree(self.workspace_root) self.worker.hold_jobs_in_build = False self.worker.release() self.waitUntilSettled() def test_post_checkout(self): project = "org/project" path = os.path.join(self.upstream_root, project) repo = git.Repo(path) repo.head.reference = repo.heads['master'] commits = [] for i in range(0, 3): commits.append(self.create_commit(project)) newRev = commits[1] cloner = zuul.lib.cloner.Cloner( git_base_url=self.upstream_root, projects=[project], workspace=self.workspace_root, zuul_branch=None, zuul_ref='master', zuul_url=self.git_root, zuul_project=project, zuul_newrev=newRev, ) cloner.execute() repos = self.getWorkspaceRepos([project]) cloned_sha = repos[project].rev_parse('HEAD').hexsha self.assertEqual(newRev, cloned_sha) def test_post_and_master_checkout(self): project = "org/project1" master_project = "org/project2" path = os.path.join(self.upstream_root, project) repo = git.Repo(path) repo.head.reference = repo.heads['master'] commits = [] for i in range(0, 3): commits.append(self.create_commit(project)) newRev = commits[1] cloner = zuul.lib.cloner.Cloner( git_base_url=self.upstream_root, projects=[project, master_project], workspace=self.workspace_root, zuul_branch=None, zuul_ref='master', zuul_url=self.git_root, zuul_project=project, zuul_newrev=newRev ) cloner.execute() repos = self.getWorkspaceRepos([project, master_project]) cloned_sha = repos[project].rev_parse('HEAD').hexsha self.assertEqual(newRev, cloned_sha) self.assertEqual( repos[master_project].rev_parse('HEAD').hexsha, repos[master_project].rev_parse('master').hexsha)