Load in-repo configuration
Change-Id: I225934407ce31f92a9b6df4bc282fbd5ec2968b3
This commit is contained in:
parent
96f2694b67
commit
14abdf44c0
|
@ -1307,7 +1307,7 @@ class ZuulTestCase(BaseTestCase):
|
||||||
# processed
|
# processed
|
||||||
self.eventQueuesJoin()
|
self.eventQueuesJoin()
|
||||||
self.sched.run_handler_lock.acquire()
|
self.sched.run_handler_lock.acquire()
|
||||||
if (not self.merge_client.build_sets and
|
if (not self.merge_client.jobs and
|
||||||
all(self.eventQueuesEmpty()) and
|
all(self.eventQueuesEmpty()) and
|
||||||
self.haveAllBuildsReported() and
|
self.haveAllBuildsReported() and
|
||||||
self.areAllBuildsWaiting()):
|
self.areAllBuildsWaiting()):
|
||||||
|
@ -1376,3 +1376,19 @@ tenants:
|
||||||
""" % os.path.abspath(path))
|
""" % os.path.abspath(path))
|
||||||
f.close()
|
f.close()
|
||||||
self.config.set('zuul', 'tenant_config', f.name)
|
self.config.set('zuul', 'tenant_config', f.name)
|
||||||
|
|
||||||
|
def addCommitToRepo(self, project, message, files, branch='master'):
|
||||||
|
path = os.path.join(self.upstream_root, project)
|
||||||
|
repo = git.Repo(path)
|
||||||
|
repo.head.reference = branch
|
||||||
|
zuul.merger.merger.reset_repo_to_head(repo)
|
||||||
|
for fn, content in files.items():
|
||||||
|
fn = os.path.join(path, fn)
|
||||||
|
with open(fn, 'w') as f:
|
||||||
|
f.write(content)
|
||||||
|
repo.index.add([fn])
|
||||||
|
commit = repo.index.commit(message)
|
||||||
|
repo.heads[branch].commit = commit
|
||||||
|
repo.head.reference = branch
|
||||||
|
repo.git.clean('-x', '-f', '-d')
|
||||||
|
repo.heads[branch].checkout()
|
||||||
|
|
|
@ -0,0 +1,36 @@
|
||||||
|
pipelines:
|
||||||
|
- name: check
|
||||||
|
manager: IndependentPipelineManager
|
||||||
|
source:
|
||||||
|
gerrit
|
||||||
|
trigger:
|
||||||
|
gerrit:
|
||||||
|
- event: patchset-created
|
||||||
|
success:
|
||||||
|
gerrit:
|
||||||
|
verified: 1
|
||||||
|
failure:
|
||||||
|
gerrit:
|
||||||
|
verified: -1
|
||||||
|
|
||||||
|
- name: tenant-one-gate
|
||||||
|
manager: DependentPipelineManager
|
||||||
|
success-message: Build succeeded (tenant-one-gate).
|
||||||
|
source:
|
||||||
|
gerrit
|
||||||
|
trigger:
|
||||||
|
gerrit:
|
||||||
|
- event: comment-added
|
||||||
|
approval:
|
||||||
|
- approved: 1
|
||||||
|
success:
|
||||||
|
gerrit:
|
||||||
|
verified: 2
|
||||||
|
submit: true
|
||||||
|
failure:
|
||||||
|
gerrit:
|
||||||
|
verified: -2
|
||||||
|
start:
|
||||||
|
gerrit:
|
||||||
|
verified: 0
|
||||||
|
precedence: high
|
|
@ -0,0 +1,8 @@
|
||||||
|
tenants:
|
||||||
|
- name: tenant-one
|
||||||
|
include:
|
||||||
|
- common.yaml
|
||||||
|
source:
|
||||||
|
gerrit:
|
||||||
|
repos:
|
||||||
|
- org/project
|
|
@ -0,0 +1,36 @@
|
||||||
|
[gearman]
|
||||||
|
server=127.0.0.1
|
||||||
|
|
||||||
|
[zuul]
|
||||||
|
tenant_config=tests/fixtures/config/in-repo/main.yaml
|
||||||
|
url_pattern=http://logs.example.com/{change.number}/{change.patchset}/{pipeline.name}/{job.name}/{build.number}
|
||||||
|
job_name_in_report=true
|
||||||
|
|
||||||
|
[merger]
|
||||||
|
git_dir=/tmp/zuul-test/git
|
||||||
|
git_user_email=zuul@example.com
|
||||||
|
git_user_name=zuul
|
||||||
|
zuul_url=http://zuul.example.com/p
|
||||||
|
|
||||||
|
[swift]
|
||||||
|
authurl=https://identity.api.example.org/v2.0/
|
||||||
|
user=username
|
||||||
|
key=password
|
||||||
|
tenant_name=" "
|
||||||
|
|
||||||
|
default_container=logs
|
||||||
|
region_name=EXP
|
||||||
|
logserver_prefix=http://logs.example.org/server.app/
|
||||||
|
|
||||||
|
[connection gerrit]
|
||||||
|
driver=gerrit
|
||||||
|
server=review.example.com
|
||||||
|
user=jenkins
|
||||||
|
sshkey=none
|
||||||
|
|
||||||
|
[connection smtp]
|
||||||
|
driver=smtp
|
||||||
|
server=localhost
|
||||||
|
port=25
|
||||||
|
default_from=zuul@example.com
|
||||||
|
default_to=you@example.com
|
|
@ -15,6 +15,7 @@
|
||||||
# under the License.
|
# under the License.
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
|
import textwrap
|
||||||
|
|
||||||
from tests.base import (
|
from tests.base import (
|
||||||
ZuulTestCase,
|
ZuulTestCase,
|
||||||
|
@ -62,3 +63,30 @@ class TestV3(ZuulTestCase):
|
||||||
|
|
||||||
self.assertEqual(A.reported, 2, "Activity in tenant two should"
|
self.assertEqual(A.reported, 2, "Activity in tenant two should"
|
||||||
"not affect tenant one")
|
"not affect tenant one")
|
||||||
|
|
||||||
|
def test_in_repo_config(self):
|
||||||
|
in_repo_conf = textwrap.dedent(
|
||||||
|
"""
|
||||||
|
projects:
|
||||||
|
- name: org/project
|
||||||
|
tenant-one-gate:
|
||||||
|
- project-test1
|
||||||
|
""")
|
||||||
|
|
||||||
|
self.addCommitToRepo('org/project', 'add zuul conf',
|
||||||
|
{'.zuul.yaml': in_repo_conf})
|
||||||
|
|
||||||
|
self.setup_config('config/in-repo/zuul.conf')
|
||||||
|
self.sched.reconfigure(self.config)
|
||||||
|
|
||||||
|
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
|
||||||
|
A.addApproval('CRVW', 2)
|
||||||
|
self.fake_gerrit.addEvent(A.addApproval('APRV', 1))
|
||||||
|
self.waitUntilSettled()
|
||||||
|
self.assertEqual(self.getJobFromHistory('project-test1').result,
|
||||||
|
'SUCCESS')
|
||||||
|
self.assertEqual(A.data['status'], 'MERGED')
|
||||||
|
self.assertEqual(A.reported, 2,
|
||||||
|
"A should report start and success")
|
||||||
|
self.assertIn('tenant-one-gate', A.messages[1],
|
||||||
|
"A should transit tenant-one gate")
|
||||||
|
|
|
@ -37,7 +37,7 @@ class ConfigSchema(object):
|
||||||
|
|
||||||
def validateTenantSource(self, value, path=[]):
|
def validateTenantSource(self, value, path=[]):
|
||||||
# TODOv3(jeblair): validate against connections
|
# TODOv3(jeblair): validate against connections
|
||||||
self.tenant_source.schema(value)
|
self.tenant_source(value)
|
||||||
|
|
||||||
def getSchema(self, data, connections=None):
|
def getSchema(self, data, connections=None):
|
||||||
tenant = {v.Required('name'): str,
|
tenant = {v.Required('name'): str,
|
||||||
|
|
|
@ -14,6 +14,7 @@
|
||||||
|
|
||||||
import json
|
import json
|
||||||
import logging
|
import logging
|
||||||
|
import threading
|
||||||
from uuid import uuid4
|
from uuid import uuid4
|
||||||
|
|
||||||
import gear
|
import gear
|
||||||
|
@ -55,6 +56,18 @@ class MergeGearmanClient(gear.Client):
|
||||||
self.__merge_client.onBuildCompleted(job)
|
self.__merge_client.onBuildCompleted(job)
|
||||||
|
|
||||||
|
|
||||||
|
class MergeJob(gear.Job):
|
||||||
|
def __init__(self, *args, **kw):
|
||||||
|
super(MergeJob, self).__init__(*args, **kw)
|
||||||
|
self.__event = threading.Event()
|
||||||
|
|
||||||
|
def setComplete(self):
|
||||||
|
self.__event.set()
|
||||||
|
|
||||||
|
def wait(self, timeout=300):
|
||||||
|
return self.__event.wait(timeout)
|
||||||
|
|
||||||
|
|
||||||
class MergeClient(object):
|
class MergeClient(object):
|
||||||
log = logging.getLogger("zuul.MergeClient")
|
log = logging.getLogger("zuul.MergeClient")
|
||||||
|
|
||||||
|
@ -71,26 +84,28 @@ class MergeClient(object):
|
||||||
self.gearman.addServer(server, port)
|
self.gearman.addServer(server, port)
|
||||||
self.log.debug("Waiting for gearman")
|
self.log.debug("Waiting for gearman")
|
||||||
self.gearman.waitForServer()
|
self.gearman.waitForServer()
|
||||||
self.build_sets = {}
|
self.jobs = set()
|
||||||
|
|
||||||
def stop(self):
|
def stop(self):
|
||||||
self.gearman.shutdown()
|
self.gearman.shutdown()
|
||||||
|
|
||||||
def areMergesOutstanding(self):
|
def areMergesOutstanding(self):
|
||||||
if self.build_sets:
|
if self.jobs:
|
||||||
return True
|
return True
|
||||||
return False
|
return False
|
||||||
|
|
||||||
def submitJob(self, name, data, build_set,
|
def submitJob(self, name, data, build_set,
|
||||||
precedence=zuul.model.PRECEDENCE_NORMAL):
|
precedence=zuul.model.PRECEDENCE_NORMAL):
|
||||||
uuid = str(uuid4().hex)
|
uuid = str(uuid4().hex)
|
||||||
job = gear.Job(name,
|
job = MergeJob(name,
|
||||||
json.dumps(data),
|
json.dumps(data),
|
||||||
unique=uuid)
|
unique=uuid)
|
||||||
|
job.build_set = build_set
|
||||||
self.log.debug("Submitting job %s with data %s" % (job, data))
|
self.log.debug("Submitting job %s with data %s" % (job, data))
|
||||||
self.build_sets[uuid] = build_set
|
self.jobs.add(job)
|
||||||
self.gearman.submitJob(job, precedence=precedence,
|
self.gearman.submitJob(job, precedence=precedence,
|
||||||
timeout=300)
|
timeout=300)
|
||||||
|
return job
|
||||||
|
|
||||||
def mergeChanges(self, items, build_set,
|
def mergeChanges(self, items, build_set,
|
||||||
precedence=zuul.model.PRECEDENCE_NORMAL):
|
precedence=zuul.model.PRECEDENCE_NORMAL):
|
||||||
|
@ -103,21 +118,29 @@ class MergeClient(object):
|
||||||
url=url)
|
url=url)
|
||||||
self.submitJob('merger:update', data, build_set, precedence)
|
self.submitJob('merger:update', data, build_set, precedence)
|
||||||
|
|
||||||
|
def getFiles(self, project, url, branch, files,
|
||||||
|
precedence=zuul.model.PRECEDENCE_HIGH):
|
||||||
|
data = dict(project=project,
|
||||||
|
url=url,
|
||||||
|
branch=branch,
|
||||||
|
files=files)
|
||||||
|
job = self.submitJob('merger:cat', data, None, precedence)
|
||||||
|
return job
|
||||||
|
|
||||||
def onBuildCompleted(self, job):
|
def onBuildCompleted(self, job):
|
||||||
build_set = self.build_sets.get(job.unique)
|
data = getJobData(job)
|
||||||
if build_set:
|
zuul_url = data.get('zuul_url')
|
||||||
data = getJobData(job)
|
merged = data.get('merged', False)
|
||||||
zuul_url = data.get('zuul_url')
|
updated = data.get('updated', False)
|
||||||
merged = data.get('merged', False)
|
commit = data.get('commit')
|
||||||
updated = data.get('updated', False)
|
job.files = data.get('files', {})
|
||||||
commit = data.get('commit')
|
self.log.info("Merge %s complete, merged: %s, updated: %s, "
|
||||||
self.log.info("Merge %s complete, merged: %s, updated: %s, "
|
"commit: %s" %
|
||||||
"commit: %s" %
|
(job, merged, updated, commit))
|
||||||
(job, merged, updated, build_set.commit))
|
job.setComplete()
|
||||||
self.sched.onMergeCompleted(build_set, zuul_url,
|
if job.build_set:
|
||||||
|
self.sched.onMergeCompleted(job.build_set, zuul_url,
|
||||||
merged, updated, commit)
|
merged, updated, commit)
|
||||||
# The test suite expects the build_set to be removed from
|
# The test suite expects the job to be removed from the
|
||||||
# the internal dict after the wake flag is set.
|
# internal account after the wake flag is set.
|
||||||
del self.build_sets[job.unique]
|
self.jobs.remove(job)
|
||||||
else:
|
|
||||||
self.log.error("Unable to find build set for uuid %s" % job.unique)
|
|
||||||
|
|
|
@ -184,6 +184,17 @@ class Repo(object):
|
||||||
origin = repo.remotes.origin
|
origin = repo.remotes.origin
|
||||||
origin.update()
|
origin.update()
|
||||||
|
|
||||||
|
def getFiles(self, branch, files):
|
||||||
|
ret = {}
|
||||||
|
repo = self.createRepoObject()
|
||||||
|
for fn in files:
|
||||||
|
tree = repo.heads[branch].commit.tree
|
||||||
|
if fn in tree:
|
||||||
|
ret[fn] = tree[fn].data_stream.read()
|
||||||
|
else:
|
||||||
|
ret[fn] = None
|
||||||
|
return ret
|
||||||
|
|
||||||
|
|
||||||
class Merger(object):
|
class Merger(object):
|
||||||
log = logging.getLogger("zuul.Merger")
|
log = logging.getLogger("zuul.Merger")
|
||||||
|
@ -342,3 +353,7 @@ class Merger(object):
|
||||||
if not commit:
|
if not commit:
|
||||||
return None
|
return None
|
||||||
return commit.hexsha
|
return commit.hexsha
|
||||||
|
|
||||||
|
def getFiles(self, project, url, branch, files):
|
||||||
|
repo = self.getRepo(project, url)
|
||||||
|
return repo.getFiles(branch, files)
|
||||||
|
|
|
@ -68,6 +68,7 @@ class MergeServer(object):
|
||||||
def register(self):
|
def register(self):
|
||||||
self.worker.registerFunction("merger:merge")
|
self.worker.registerFunction("merger:merge")
|
||||||
self.worker.registerFunction("merger:update")
|
self.worker.registerFunction("merger:update")
|
||||||
|
self.worker.registerFunction("merger:cat")
|
||||||
|
|
||||||
def stop(self):
|
def stop(self):
|
||||||
self.log.debug("Stopping")
|
self.log.debug("Stopping")
|
||||||
|
@ -90,6 +91,9 @@ class MergeServer(object):
|
||||||
elif job.name == 'merger:update':
|
elif job.name == 'merger:update':
|
||||||
self.log.debug("Got update job: %s" % job.unique)
|
self.log.debug("Got update job: %s" % job.unique)
|
||||||
self.update(job)
|
self.update(job)
|
||||||
|
elif job.name == 'merger:cat':
|
||||||
|
self.log.debug("Got cat job: %s" % job.unique)
|
||||||
|
self.cat(job)
|
||||||
else:
|
else:
|
||||||
self.log.error("Unable to handle job %s" % job.name)
|
self.log.error("Unable to handle job %s" % job.name)
|
||||||
job.sendWorkFail()
|
job.sendWorkFail()
|
||||||
|
@ -113,3 +117,13 @@ class MergeServer(object):
|
||||||
result = dict(updated=True,
|
result = dict(updated=True,
|
||||||
zuul_url=self.zuul_url)
|
zuul_url=self.zuul_url)
|
||||||
job.sendWorkComplete(json.dumps(result))
|
job.sendWorkComplete(json.dumps(result))
|
||||||
|
|
||||||
|
def cat(self, job):
|
||||||
|
args = json.loads(job.arguments)
|
||||||
|
self.merger.updateRepo(args['project'], args['url'])
|
||||||
|
files = self.merger.getFiles(args['project'], args['url'],
|
||||||
|
args['branch'], args['files'])
|
||||||
|
result = dict(updated=True,
|
||||||
|
files=files,
|
||||||
|
zuul_url=self.zuul_url)
|
||||||
|
job.sendWorkComplete(json.dumps(result))
|
||||||
|
|
|
@ -353,6 +353,7 @@ class Scheduler(threading.Thread):
|
||||||
raise Exception("Unable to read tenant config file at %s" %
|
raise Exception("Unable to read tenant config file at %s" %
|
||||||
config_path)
|
config_path)
|
||||||
with open(config_path) as config_file:
|
with open(config_path) as config_file:
|
||||||
|
self.log.info("Loading configuration from %s" % (config_path,))
|
||||||
data = yaml.load(config_file)
|
data = yaml.load(config_file)
|
||||||
base = os.path.dirname(os.path.realpath(config_path))
|
base = os.path.dirname(os.path.realpath(config_path))
|
||||||
|
|
||||||
|
@ -368,8 +369,11 @@ class Scheduler(threading.Thread):
|
||||||
fn = os.path.join(base, fn)
|
fn = os.path.join(base, fn)
|
||||||
fn = os.path.expanduser(fn)
|
fn = os.path.expanduser(fn)
|
||||||
with open(fn) as config_file:
|
with open(fn) as config_file:
|
||||||
|
self.log.info("Loading configuration from %s" % (fn,))
|
||||||
incdata = yaml.load(config_file)
|
incdata = yaml.load(config_file)
|
||||||
extend_dict(tenant_config, incdata)
|
extend_dict(tenant_config, incdata)
|
||||||
|
incdata = self._parseTenantInRepoLayouts(conf_tenant)
|
||||||
|
extend_dict(tenant_config, incdata)
|
||||||
tenant.layout = self._parseLayout(base, tenant_config, connections)
|
tenant.layout = self._parseLayout(base, tenant_config, connections)
|
||||||
return abide
|
return abide
|
||||||
|
|
||||||
|
@ -583,6 +587,39 @@ class Scheduler(threading.Thread):
|
||||||
|
|
||||||
return layout
|
return layout
|
||||||
|
|
||||||
|
def _parseTenantInRepoLayouts(self, conf_tenant):
|
||||||
|
config = {}
|
||||||
|
jobs = []
|
||||||
|
for source_name, conf_source in conf_tenant.get('source', {}).items():
|
||||||
|
# TODOv3(jeblair,jhesketh): sources should just be
|
||||||
|
# set up at the start of the zuul.conf parsing
|
||||||
|
if source_name not in self.sources:
|
||||||
|
self.sources[source_name] = self._getSourceDriver(
|
||||||
|
source_name)
|
||||||
|
for conf_repo in conf_source.get('repos'):
|
||||||
|
source = self.sources[source_name]
|
||||||
|
project = source.getProject(conf_repo)
|
||||||
|
url = source.getGitUrl(project)
|
||||||
|
# TODOv3(jeblair): config should be branch specific
|
||||||
|
job = self.merger.getFiles(project.name, url, 'master',
|
||||||
|
files=['.zuul.yaml'])
|
||||||
|
job.project = project
|
||||||
|
jobs.append(job)
|
||||||
|
for job in jobs:
|
||||||
|
self.log.debug("Waiting for cat job %s" % (job,))
|
||||||
|
job.wait()
|
||||||
|
if job.files.get('.zuul.yaml'):
|
||||||
|
self.log.info("Loading configuration from %s/.zuul.yaml" %
|
||||||
|
(job.project,))
|
||||||
|
incdata = self._parseInRepoLayout(job.files['.zuul.yaml'])
|
||||||
|
extend_dict(config, incdata)
|
||||||
|
return config
|
||||||
|
|
||||||
|
def _parseInRepoLayout(self, data):
|
||||||
|
# TODOv3(jeblair): this should implement some rules to protect
|
||||||
|
# aspects of the config that should not be changed in-repo
|
||||||
|
return yaml.load(data)
|
||||||
|
|
||||||
def setLauncher(self, launcher):
|
def setLauncher(self, launcher):
|
||||||
self.launcher = launcher
|
self.launcher = launcher
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue