zuul/tests/test_cloner.py
Joshua Hesketh 29d99b7eb1 Fix pep8 issues
The pep8 jobs aren't running in gate at the moment.
Fix up the pep8 issues and remove the tox filter that stopped it
from running.

Also ignore E129 (visually indented line with same indent as next
logical line) as we don't follow it.

Change-Id: I394708ba96797bbc6fcd951e6436a104be0a3746
2015-01-21 19:38:33 +11:00

569 lines
25 KiB
Python

#!/usr/bin/env python
# Copyright 2012 Hewlett-Packard Development Company, L.P.
# Copyright 2014 Wikimedia Foundation Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import logging
import os
import shutil
import time
import git
import zuul.lib.cloner
from tests.base import ZuulTestCase
logging.basicConfig(level=logging.DEBUG,
format='%(asctime)s %(name)-32s '
'%(levelname)-8s %(message)s')
class TestCloner(ZuulTestCase):
log = logging.getLogger("zuul.test.cloner")
workspace_root = None
def setUp(self):
super(TestCloner, self).setUp()
self.workspace_root = os.path.join(self.test_root, 'workspace')
self.config.set('zuul', 'layout_config',
'tests/fixtures/layout-cloner.yaml')
self.sched.reconfigure(self.config)
self.registerJobs()
def getWorkspaceRepos(self, projects):
repos = {}
for project in projects:
repos[project] = git.Repo(
os.path.join(self.workspace_root, project))
return repos
def getUpstreamRepos(self, projects):
repos = {}
for project in projects:
repos[project] = git.Repo(
os.path.join(self.upstream_root, project))
return repos
def test_cache_dir(self):
projects = ['org/project1', 'org/project2']
cache_root = os.path.join(self.test_root, "cache")
for project in projects:
upstream_repo_path = os.path.join(self.upstream_root, project)
cache_repo_path = os.path.join(cache_root, project)
git.Repo.clone_from(upstream_repo_path, cache_repo_path)
self.worker.hold_jobs_in_build = True
A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
A.addApproval('CRVW', 2)
self.fake_gerrit.addEvent(A.addApproval('APRV', 1))
self.waitUntilSettled()
self.assertEquals(1, len(self.builds), "One build is running")
B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
B.setMerged()
upstream = self.getUpstreamRepos(projects)
states = [{
'org/project1': self.builds[0].parameters['ZUUL_COMMIT'],
'org/project2': str(upstream['org/project2'].commit('master')),
}]
for number, build in enumerate(self.builds):
self.log.debug("Build parameters: %s", build.parameters)
cloner = zuul.lib.cloner.Cloner(
git_base_url=self.upstream_root,
projects=projects,
workspace=self.workspace_root,
zuul_branch=build.parameters['ZUUL_BRANCH'],
zuul_ref=build.parameters['ZUUL_REF'],
zuul_url=self.git_root,
cache_dir=cache_root,
)
cloner.execute()
work = self.getWorkspaceRepos(projects)
state = states[number]
for project in projects:
self.assertEquals(state[project],
str(work[project].commit('HEAD')),
'Project %s commit for build %s should '
'be correct' % (project, number))
work = self.getWorkspaceRepos(projects)
upstream_repo_path = os.path.join(self.upstream_root, 'org/project1')
self.assertEquals(
work['org/project1'].remotes.origin.url,
upstream_repo_path,
'workspace repo origin should be upstream, not cache'
)
self.worker.hold_jobs_in_build = False
self.worker.release()
self.waitUntilSettled()
def test_one_branch(self):
self.worker.hold_jobs_in_build = True
projects = ['org/project1', 'org/project2']
A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
A.addApproval('CRVW', 2)
B.addApproval('CRVW', 2)
self.fake_gerrit.addEvent(A.addApproval('APRV', 1))
self.fake_gerrit.addEvent(B.addApproval('APRV', 1))
self.waitUntilSettled()
self.assertEquals(2, len(self.builds), "Two builds are running")
upstream = self.getUpstreamRepos(projects)
states = [
{'org/project1': self.builds[0].parameters['ZUUL_COMMIT'],
'org/project2': str(upstream['org/project2'].commit('master')),
},
{'org/project1': self.builds[0].parameters['ZUUL_COMMIT'],
'org/project2': self.builds[1].parameters['ZUUL_COMMIT'],
},
]
for number, build in enumerate(self.builds):
self.log.debug("Build parameters: %s", build.parameters)
cloner = zuul.lib.cloner.Cloner(
git_base_url=self.upstream_root,
projects=projects,
workspace=self.workspace_root,
zuul_branch=build.parameters['ZUUL_BRANCH'],
zuul_ref=build.parameters['ZUUL_REF'],
zuul_url=self.git_root,
)
cloner.execute()
work = self.getWorkspaceRepos(projects)
state = states[number]
for project in projects:
self.assertEquals(state[project],
str(work[project].commit('HEAD')),
'Project %s commit for build %s should '
'be correct' % (project, number))
shutil.rmtree(self.workspace_root)
self.worker.hold_jobs_in_build = False
self.worker.release()
self.waitUntilSettled()
def test_multi_branch(self):
self.worker.hold_jobs_in_build = True
projects = ['org/project1', 'org/project2',
'org/project3', 'org/project4']
self.create_branch('org/project2', 'stable/havana')
self.create_branch('org/project4', 'stable/havana')
A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
B = self.fake_gerrit.addFakeChange('org/project2', 'stable/havana',
'B')
C = self.fake_gerrit.addFakeChange('org/project3', 'master', 'C')
A.addApproval('CRVW', 2)
B.addApproval('CRVW', 2)
C.addApproval('CRVW', 2)
self.fake_gerrit.addEvent(A.addApproval('APRV', 1))
self.fake_gerrit.addEvent(B.addApproval('APRV', 1))
self.fake_gerrit.addEvent(C.addApproval('APRV', 1))
self.waitUntilSettled()
self.assertEquals(3, len(self.builds), "Three builds are running")
upstream = self.getUpstreamRepos(projects)
states = [
{'org/project1': self.builds[0].parameters['ZUUL_COMMIT'],
'org/project2': str(upstream['org/project2'].commit('master')),
'org/project3': str(upstream['org/project3'].commit('master')),
'org/project4': str(upstream['org/project4'].
commit('master')),
},
{'org/project1': self.builds[0].parameters['ZUUL_COMMIT'],
'org/project2': self.builds[1].parameters['ZUUL_COMMIT'],
'org/project3': str(upstream['org/project3'].commit('master')),
'org/project4': str(upstream['org/project4'].
commit('stable/havana')),
},
{'org/project1': self.builds[0].parameters['ZUUL_COMMIT'],
'org/project2': str(upstream['org/project2'].commit('master')),
'org/project3': self.builds[2].parameters['ZUUL_COMMIT'],
'org/project4': str(upstream['org/project4'].
commit('master')),
},
]
for number, build in enumerate(self.builds):
self.log.debug("Build parameters: %s", build.parameters)
cloner = zuul.lib.cloner.Cloner(
git_base_url=self.upstream_root,
projects=projects,
workspace=self.workspace_root,
zuul_branch=build.parameters['ZUUL_BRANCH'],
zuul_ref=build.parameters['ZUUL_REF'],
zuul_url=self.git_root,
)
cloner.execute()
work = self.getWorkspaceRepos(projects)
state = states[number]
for project in projects:
self.assertEquals(state[project],
str(work[project].commit('HEAD')),
'Project %s commit for build %s should '
'be correct' % (project, number))
shutil.rmtree(self.workspace_root)
self.worker.hold_jobs_in_build = False
self.worker.release()
self.waitUntilSettled()
def test_upgrade(self):
# Simulates an upgrade test
self.worker.hold_jobs_in_build = True
projects = ['org/project1', 'org/project2', 'org/project3',
'org/project4', 'org/project5', 'org/project6']
self.create_branch('org/project2', 'stable/havana')
self.create_branch('org/project3', 'stable/havana')
self.create_branch('org/project4', 'stable/havana')
self.create_branch('org/project5', 'stable/havana')
A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
C = self.fake_gerrit.addFakeChange('org/project3', 'stable/havana',
'C')
D = self.fake_gerrit.addFakeChange('org/project3', 'master', 'D')
E = self.fake_gerrit.addFakeChange('org/project4', 'stable/havana',
'E')
A.addApproval('CRVW', 2)
B.addApproval('CRVW', 2)
C.addApproval('CRVW', 2)
D.addApproval('CRVW', 2)
E.addApproval('CRVW', 2)
self.fake_gerrit.addEvent(A.addApproval('APRV', 1))
self.fake_gerrit.addEvent(B.addApproval('APRV', 1))
self.fake_gerrit.addEvent(C.addApproval('APRV', 1))
self.fake_gerrit.addEvent(D.addApproval('APRV', 1))
self.fake_gerrit.addEvent(E.addApproval('APRV', 1))
self.waitUntilSettled()
self.assertEquals(5, len(self.builds), "Five builds are running")
# Check the old side of the upgrade first
upstream = self.getUpstreamRepos(projects)
states = [
{'org/project1': self.builds[0].parameters['ZUUL_COMMIT'],
'org/project2': str(upstream['org/project2'].commit(
'stable/havana')),
'org/project3': str(upstream['org/project3'].commit(
'stable/havana')),
'org/project4': str(upstream['org/project4'].commit(
'stable/havana')),
'org/project5': str(upstream['org/project5'].commit(
'stable/havana')),
'org/project6': str(upstream['org/project6'].commit('master')),
},
{'org/project1': self.builds[0].parameters['ZUUL_COMMIT'],
'org/project2': str(upstream['org/project2'].commit(
'stable/havana')),
'org/project3': str(upstream['org/project3'].commit(
'stable/havana')),
'org/project4': str(upstream['org/project4'].commit(
'stable/havana')),
'org/project5': str(upstream['org/project5'].commit(
'stable/havana')),
'org/project6': str(upstream['org/project6'].commit('master')),
},
{'org/project1': self.builds[0].parameters['ZUUL_COMMIT'],
'org/project2': str(upstream['org/project2'].commit(
'stable/havana')),
'org/project3': self.builds[2].parameters['ZUUL_COMMIT'],
'org/project4': str(upstream['org/project4'].commit(
'stable/havana')),
'org/project5': str(upstream['org/project5'].commit(
'stable/havana')),
'org/project6': str(upstream['org/project6'].commit('master')),
},
{'org/project1': self.builds[0].parameters['ZUUL_COMMIT'],
'org/project2': str(upstream['org/project2'].commit(
'stable/havana')),
'org/project3': self.builds[2].parameters['ZUUL_COMMIT'],
'org/project4': str(upstream['org/project4'].commit(
'stable/havana')),
'org/project5': str(upstream['org/project5'].commit(
'stable/havana')),
'org/project6': str(upstream['org/project6'].commit('master')),
},
{'org/project1': self.builds[0].parameters['ZUUL_COMMIT'],
'org/project2': str(upstream['org/project2'].commit(
'stable/havana')),
'org/project3': self.builds[2].parameters['ZUUL_COMMIT'],
'org/project4': self.builds[4].parameters['ZUUL_COMMIT'],
'org/project5': str(upstream['org/project5'].commit(
'stable/havana')),
'org/project6': str(upstream['org/project6'].commit('master')),
},
]
for number, build in enumerate(self.builds):
self.log.debug("Build parameters: %s", build.parameters)
cloner = zuul.lib.cloner.Cloner(
git_base_url=self.upstream_root,
projects=projects,
workspace=self.workspace_root,
zuul_branch=build.parameters['ZUUL_BRANCH'],
zuul_ref=build.parameters['ZUUL_REF'],
zuul_url=self.git_root,
branch='stable/havana', # Old branch for upgrade
)
cloner.execute()
work = self.getWorkspaceRepos(projects)
state = states[number]
for project in projects:
self.assertEquals(state[project],
str(work[project].commit('HEAD')),
'Project %s commit for build %s should '
'be correct on old side of upgrade' %
(project, number))
shutil.rmtree(self.workspace_root)
# Check the new side of the upgrade
states = [
{'org/project1': self.builds[0].parameters['ZUUL_COMMIT'],
'org/project2': str(upstream['org/project2'].commit('master')),
'org/project3': str(upstream['org/project3'].commit('master')),
'org/project4': str(upstream['org/project4'].commit('master')),
'org/project5': str(upstream['org/project5'].commit('master')),
'org/project6': str(upstream['org/project6'].commit('master')),
},
{'org/project1': self.builds[0].parameters['ZUUL_COMMIT'],
'org/project2': self.builds[1].parameters['ZUUL_COMMIT'],
'org/project3': str(upstream['org/project3'].commit('master')),
'org/project4': str(upstream['org/project4'].commit('master')),
'org/project5': str(upstream['org/project5'].commit('master')),
'org/project6': str(upstream['org/project6'].commit('master')),
},
{'org/project1': self.builds[0].parameters['ZUUL_COMMIT'],
'org/project2': self.builds[1].parameters['ZUUL_COMMIT'],
'org/project3': str(upstream['org/project3'].commit('master')),
'org/project4': str(upstream['org/project4'].commit('master')),
'org/project5': str(upstream['org/project5'].commit('master')),
'org/project6': str(upstream['org/project6'].commit('master')),
},
{'org/project1': self.builds[0].parameters['ZUUL_COMMIT'],
'org/project2': self.builds[1].parameters['ZUUL_COMMIT'],
'org/project3': self.builds[3].parameters['ZUUL_COMMIT'],
'org/project4': str(upstream['org/project4'].commit('master')),
'org/project5': str(upstream['org/project5'].commit('master')),
'org/project6': str(upstream['org/project6'].commit('master')),
},
{'org/project1': self.builds[0].parameters['ZUUL_COMMIT'],
'org/project2': self.builds[1].parameters['ZUUL_COMMIT'],
'org/project3': self.builds[3].parameters['ZUUL_COMMIT'],
'org/project4': str(upstream['org/project4'].commit('master')),
'org/project5': str(upstream['org/project5'].commit('master')),
'org/project6': str(upstream['org/project6'].commit('master')),
},
]
for number, build in enumerate(self.builds):
self.log.debug("Build parameters: %s", build.parameters)
cloner = zuul.lib.cloner.Cloner(
git_base_url=self.upstream_root,
projects=projects,
workspace=self.workspace_root,
zuul_branch=build.parameters['ZUUL_BRANCH'],
zuul_ref=build.parameters['ZUUL_REF'],
zuul_url=self.git_root,
branch='master', # New branch for upgrade
)
cloner.execute()
work = self.getWorkspaceRepos(projects)
state = states[number]
for project in projects:
self.assertEquals(state[project],
str(work[project].commit('HEAD')),
'Project %s commit for build %s should '
'be correct on old side of upgrade' %
(project, number))
shutil.rmtree(self.workspace_root)
self.worker.hold_jobs_in_build = False
self.worker.release()
self.waitUntilSettled()
def test_project_override(self):
self.worker.hold_jobs_in_build = True
projects = ['org/project1', 'org/project2', 'org/project3',
'org/project4', 'org/project5', 'org/project6']
self.create_branch('org/project3', 'stable/havana')
self.create_branch('org/project4', 'stable/havana')
self.create_branch('org/project6', 'stable/havana')
A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B')
C = self.fake_gerrit.addFakeChange('org/project2', 'master', 'C')
D = self.fake_gerrit.addFakeChange('org/project3', 'stable/havana',
'D')
A.addApproval('CRVW', 2)
B.addApproval('CRVW', 2)
C.addApproval('CRVW', 2)
D.addApproval('CRVW', 2)
self.fake_gerrit.addEvent(A.addApproval('APRV', 1))
self.fake_gerrit.addEvent(B.addApproval('APRV', 1))
self.fake_gerrit.addEvent(C.addApproval('APRV', 1))
self.fake_gerrit.addEvent(D.addApproval('APRV', 1))
self.waitUntilSettled()
self.assertEquals(4, len(self.builds), "Four builds are running")
upstream = self.getUpstreamRepos(projects)
states = [
{'org/project1': self.builds[0].parameters['ZUUL_COMMIT'],
'org/project2': str(upstream['org/project2'].commit('master')),
'org/project3': str(upstream['org/project3'].commit('master')),
'org/project4': str(upstream['org/project4'].commit('master')),
'org/project5': str(upstream['org/project5'].commit('master')),
'org/project6': str(upstream['org/project6'].commit('master')),
},
{'org/project1': self.builds[1].parameters['ZUUL_COMMIT'],
'org/project2': str(upstream['org/project2'].commit('master')),
'org/project3': str(upstream['org/project3'].commit('master')),
'org/project4': str(upstream['org/project4'].commit('master')),
'org/project5': str(upstream['org/project5'].commit('master')),
'org/project6': str(upstream['org/project6'].commit('master')),
},
{'org/project1': self.builds[1].parameters['ZUUL_COMMIT'],
'org/project2': self.builds[2].parameters['ZUUL_COMMIT'],
'org/project3': str(upstream['org/project3'].commit('master')),
'org/project4': str(upstream['org/project4'].commit('master')),
'org/project5': str(upstream['org/project5'].commit('master')),
'org/project6': str(upstream['org/project6'].commit('master')),
},
{'org/project1': self.builds[1].parameters['ZUUL_COMMIT'],
'org/project2': self.builds[2].parameters['ZUUL_COMMIT'],
'org/project3': self.builds[3].parameters['ZUUL_COMMIT'],
'org/project4': str(upstream['org/project4'].commit('master')),
'org/project5': str(upstream['org/project5'].commit('master')),
'org/project6': str(upstream['org/project6'].commit(
'stable/havana')),
},
]
for number, build in enumerate(self.builds):
self.log.debug("Build parameters: %s", build.parameters)
cloner = zuul.lib.cloner.Cloner(
git_base_url=self.upstream_root,
projects=projects,
workspace=self.workspace_root,
zuul_branch=build.parameters['ZUUL_BRANCH'],
zuul_ref=build.parameters['ZUUL_REF'],
zuul_url=self.git_root,
project_branches={'org/project4': 'master'},
)
cloner.execute()
work = self.getWorkspaceRepos(projects)
state = states[number]
for project in projects:
self.assertEquals(state[project],
str(work[project].commit('HEAD')),
'Project %s commit for build %s should '
'be correct' % (project, number))
shutil.rmtree(self.workspace_root)
self.worker.hold_jobs_in_build = False
self.worker.release()
self.waitUntilSettled()
def test_periodic(self):
self.worker.hold_jobs_in_build = True
self.create_branch('org/project', 'stable/havana')
self.config.set('zuul', 'layout_config',
'tests/fixtures/layout-timer.yaml')
self.sched.reconfigure(self.config)
self.registerJobs()
# The pipeline triggers every second, so we should have seen
# several by now.
time.sleep(5)
self.waitUntilSettled()
builds = self.builds[:]
self.worker.hold_jobs_in_build = False
# Stop queuing timer triggered jobs so that the assertions
# below don't race against more jobs being queued.
self.config.set('zuul', 'layout_config',
'tests/fixtures/layout-no-timer.yaml')
self.sched.reconfigure(self.config)
self.registerJobs()
self.worker.release()
self.waitUntilSettled()
projects = ['org/project']
self.assertEquals(2, len(builds), "Two builds are running")
upstream = self.getUpstreamRepos(projects)
states = [
{'org/project':
str(upstream['org/project'].commit('stable/havana')),
},
{'org/project':
str(upstream['org/project'].commit('stable/havana')),
},
]
for number, build in enumerate(builds):
self.log.debug("Build parameters: %s", build.parameters)
cloner = zuul.lib.cloner.Cloner(
git_base_url=self.upstream_root,
projects=projects,
workspace=self.workspace_root,
zuul_branch=build.parameters.get('ZUUL_BRANCH', None),
zuul_ref=build.parameters.get('ZUUL_REF', None),
zuul_url=self.git_root,
branch='stable/havana',
)
cloner.execute()
work = self.getWorkspaceRepos(projects)
state = states[number]
for project in projects:
self.assertEquals(state[project],
str(work[project].commit('HEAD')),
'Project %s commit for build %s should '
'be correct' % (project, number))
shutil.rmtree(self.workspace_root)
self.worker.hold_jobs_in_build = False
self.worker.release()
self.waitUntilSettled()