zuul
/
zuul
Code Issues Proposed changes

Load in-repo configuration 

Change-Id: I225934407ce31f92a9b6df4bc282fbd5ec2968b3
This commit is contained in:
James E. Blair 2015-12-09 16:11:53 -08:00
parent 96f2694b67
commit 14abdf44c0
10 changed files with 235 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
tests/base.py
View File

@ -1307,7 +1307,7 @@ class ZuulTestCase(BaseTestCase):
# processed
self.eventQueuesJoin()
self.sched.run_handler_lock.acquire()
if (not self.merge_client.build_sets and
if (not self.merge_client.jobs and
all(self.eventQueuesEmpty()) and
self.haveAllBuildsReported() and
self.areAllBuildsWaiting()):
@ -1376,3 +1376,19 @@ tenants:
""" % os.path.abspath(path))
f.close()
self.config.set('zuul', 'tenant_config', f.name)
def addCommitToRepo(self, project, message, files, branch='master'):
path = os.path.join(self.upstream_root, project)
repo = git.Repo(path)
repo.head.reference = branch
zuul.merger.merger.reset_repo_to_head(repo)
for fn, content in files.items():
fn = os.path.join(path, fn)
with open(fn, 'w') as f:
f.write(content)
repo.index.add([fn])
commit = repo.index.commit(message)
repo.heads[branch].commit = commit
repo.head.reference = branch
repo.git.clean('-x', '-f', '-d')
repo.heads[branch].checkout()

36
tests/fixtures/config/in-repo/common.yaml vendored Normal file
View File

@ -0,0 +1,36 @@
pipelines:
- name: check
manager: IndependentPipelineManager
source:
gerrit
trigger:
gerrit:
- event: patchset-created
success:
gerrit:
verified: 1
failure:
gerrit:
verified: -1
- name: tenant-one-gate
manager: DependentPipelineManager
success-message: Build succeeded (tenant-one-gate).
source:
gerrit
trigger:
gerrit:
- event: comment-added
approval:
- approved: 1
success:
gerrit:
verified: 2
submit: true
failure:
gerrit:
verified: -2
start:
gerrit:
verified: 0
precedence: high

8
tests/fixtures/config/in-repo/main.yaml vendored Normal file
View File

@ -0,0 +1,8 @@
tenants:
- name: tenant-one
include:
- common.yaml
source:
gerrit:
repos:
- org/project

36
tests/fixtures/config/in-repo/zuul.conf vendored Normal file
View File

@ -0,0 +1,36 @@
[gearman]
server=127.0.0.1
[zuul]
tenant_config=tests/fixtures/config/in-repo/main.yaml
url_pattern=http://logs.example.com/{change.number}/{change.patchset}/{pipeline.name}/{job.name}/{build.number}
job_name_in_report=true
[merger]
git_dir=/tmp/zuul-test/git
git_user_email=zuul@example.com
git_user_name=zuul
zuul_url=http://zuul.example.com/p
[swift]
authurl=https://identity.api.example.org/v2.0/
user=username
key=password
tenant_name=" "
default_container=logs
region_name=EXP
logserver_prefix=http://logs.example.org/server.app/
[connection gerrit]
driver=gerrit
server=review.example.com
user=jenkins
sshkey=none
[connection smtp]
driver=smtp
server=localhost
port=25
default_from=zuul@example.com
default_to=you@example.com

28
tests/test_v3.py
View File

@ -15,6 +15,7 @@
# under the License.
import logging
import textwrap
from tests.base import (
ZuulTestCase,
@ -62,3 +63,30 @@ class TestV3(ZuulTestCase):
self.assertEqual(A.reported, 2, "Activity in tenant two should"
"not affect tenant one")
def test_in_repo_config(self):
in_repo_conf = textwrap.dedent(
"""
projects:
- name: org/project
tenant-one-gate:
- project-test1
""")
self.addCommitToRepo('org/project', 'add zuul conf',
{'.zuul.yaml': in_repo_conf})
self.setup_config('config/in-repo/zuul.conf')
self.sched.reconfigure(self.config)
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
A.addApproval('CRVW', 2)
self.fake_gerrit.addEvent(A.addApproval('APRV', 1))
self.waitUntilSettled()
self.assertEqual(self.getJobFromHistory('project-test1').result,
'SUCCESS')
self.assertEqual(A.data['status'], 'MERGED')
self.assertEqual(A.reported, 2,
"A should report start and success")
self.assertIn('tenant-one-gate', A.messages[1],
"A should transit tenant-one gate")

2
zuul/layoutvalidator.py
View File

@ -37,7 +37,7 @@ class ConfigSchema(object):
def validateTenantSource(self, value, path=[]):
# TODOv3(jeblair): validate against connections
self.tenant_source.schema(value)
self.tenant_source(value)
def getSchema(self, data, connections=None):
tenant = {v.Required('name'): str,

63
zuul/merger/client.py
View File

@ -14,6 +14,7 @@
import json
import logging
import threading
from uuid import uuid4
import gear
@ -55,6 +56,18 @@ class MergeGearmanClient(gear.Client):
self.__merge_client.onBuildCompleted(job)
class MergeJob(gear.Job):
def __init__(self, *args, **kw):
super(MergeJob, self).__init__(*args, **kw)
self.__event = threading.Event()
def setComplete(self):
self.__event.set()
def wait(self, timeout=300):
return self.__event.wait(timeout)
class MergeClient(object):
log = logging.getLogger("zuul.MergeClient")
@ -71,26 +84,28 @@ class MergeClient(object):
self.gearman.addServer(server, port)
self.log.debug("Waiting for gearman")
self.gearman.waitForServer()
self.build_sets = {}
self.jobs = set()
def stop(self):
self.gearman.shutdown()
def areMergesOutstanding(self):
if self.build_sets:
if self.jobs:
return True
return False
def submitJob(self, name, data, build_set,
precedence=zuul.model.PRECEDENCE_NORMAL):
uuid = str(uuid4().hex)
job = gear.Job(name,
job = MergeJob(name,
json.dumps(data),
unique=uuid)
job.build_set = build_set
self.log.debug("Submitting job %s with data %s" % (job, data))
self.build_sets[uuid] = build_set
self.jobs.add(job)
self.gearman.submitJob(job, precedence=precedence,
timeout=300)
return job
def mergeChanges(self, items, build_set,
precedence=zuul.model.PRECEDENCE_NORMAL):
@ -103,21 +118,29 @@ class MergeClient(object):
url=url)
self.submitJob('merger:update', data, build_set, precedence)
def getFiles(self, project, url, branch, files,
precedence=zuul.model.PRECEDENCE_HIGH):
data = dict(project=project,
url=url,
branch=branch,
files=files)
job = self.submitJob('merger:cat', data, None, precedence)
return job
def onBuildCompleted(self, job):
build_set = self.build_sets.get(job.unique)
if build_set:
data = getJobData(job)
zuul_url = data.get('zuul_url')
merged = data.get('merged', False)
updated = data.get('updated', False)
commit = data.get('commit')
self.log.info("Merge %s complete, merged: %s, updated: %s, "
"commit: %s" %
(job, merged, updated, build_set.commit))
self.sched.onMergeCompleted(build_set, zuul_url,
data = getJobData(job)
zuul_url = data.get('zuul_url')
merged = data.get('merged', False)
updated = data.get('updated', False)
commit = data.get('commit')
job.files = data.get('files', {})
self.log.info("Merge %s complete, merged: %s, updated: %s, "
"commit: %s" %
(job, merged, updated, commit))
job.setComplete()
if job.build_set:
self.sched.onMergeCompleted(job.build_set, zuul_url,
merged, updated, commit)
# The test suite expects the build_set to be removed from
# the internal dict after the wake flag is set.
del self.build_sets[job.unique]
else:
self.log.error("Unable to find build set for uuid %s" % job.unique)
# The test suite expects the job to be removed from the
# internal account after the wake flag is set.
self.jobs.remove(job)

15
zuul/merger/merger.py
View File

@ -184,6 +184,17 @@ class Repo(object):
origin = repo.remotes.origin
origin.update()
def getFiles(self, branch, files):
ret = {}
repo = self.createRepoObject()
for fn in files:
tree = repo.heads[branch].commit.tree
if fn in tree:
ret[fn] = tree[fn].data_stream.read()
else:
ret[fn] = None
return ret
class Merger(object):
log = logging.getLogger("zuul.Merger")
@ -342,3 +353,7 @@ class Merger(object):
if not commit:
return None
return commit.hexsha
def getFiles(self, project, url, branch, files):
repo = self.getRepo(project, url)
return repo.getFiles(branch, files)

14
zuul/merger/server.py
View File

@ -68,6 +68,7 @@ class MergeServer(object):
def register(self):
self.worker.registerFunction("merger:merge")
self.worker.registerFunction("merger:update")
self.worker.registerFunction("merger:cat")
def stop(self):
self.log.debug("Stopping")
@ -90,6 +91,9 @@ class MergeServer(object):
elif job.name == 'merger:update':
self.log.debug("Got update job: %s" % job.unique)
self.update(job)
elif job.name == 'merger:cat':
self.log.debug("Got cat job: %s" % job.unique)
self.cat(job)
else:
self.log.error("Unable to handle job %s" % job.name)
job.sendWorkFail()
@ -113,3 +117,13 @@ class MergeServer(object):
result = dict(updated=True,
zuul_url=self.zuul_url)
job.sendWorkComplete(json.dumps(result))
def cat(self, job):
args = json.loads(job.arguments)
self.merger.updateRepo(args['project'], args['url'])
files = self.merger.getFiles(args['project'], args['url'],
args['branch'], args['files'])
result = dict(updated=True,
files=files,
zuul_url=self.zuul_url)
job.sendWorkComplete(json.dumps(result))

37
zuul/scheduler.py
View File

@ -353,6 +353,7 @@ class Scheduler(threading.Thread):
raise Exception("Unable to read tenant config file at %s" %
config_path)
with open(config_path) as config_file:
self.log.info("Loading configuration from %s" % (config_path,))
data = yaml.load(config_file)
base = os.path.dirname(os.path.realpath(config_path))
@ -368,8 +369,11 @@ class Scheduler(threading.Thread):
fn = os.path.join(base, fn)
fn = os.path.expanduser(fn)
with open(fn) as config_file:
self.log.info("Loading configuration from %s" % (fn,))
incdata = yaml.load(config_file)
extend_dict(tenant_config, incdata)
incdata = self._parseTenantInRepoLayouts(conf_tenant)
extend_dict(tenant_config, incdata)
tenant.layout = self._parseLayout(base, tenant_config, connections)
return abide
@ -583,6 +587,39 @@ class Scheduler(threading.Thread):
return layout
def _parseTenantInRepoLayouts(self, conf_tenant):
config = {}
jobs = []
for source_name, conf_source in conf_tenant.get('source', {}).items():
# TODOv3(jeblair,jhesketh): sources should just be
# set up at the start of the zuul.conf parsing
if source_name not in self.sources:
self.sources[source_name] = self._getSourceDriver(
source_name)
for conf_repo in conf_source.get('repos'):
source = self.sources[source_name]
project = source.getProject(conf_repo)
url = source.getGitUrl(project)
# TODOv3(jeblair): config should be branch specific
job = self.merger.getFiles(project.name, url, 'master',
files=['.zuul.yaml'])
job.project = project
jobs.append(job)
for job in jobs:
self.log.debug("Waiting for cat job %s" % (job,))
job.wait()
if job.files.get('.zuul.yaml'):
self.log.info("Loading configuration from %s/.zuul.yaml" %
(job.project,))
incdata = self._parseInRepoLayout(job.files['.zuul.yaml'])
extend_dict(config, incdata)
return config
def _parseInRepoLayout(self, data):
# TODOv3(jeblair): this should implement some rules to protect
# aspects of the config that should not be changed in-repo
return yaml.load(data)
def setLauncher(self, launcher):
self.launcher = launcher