Load in-repo configuration
Change-Id: I225934407ce31f92a9b6df4bc282fbd5ec2968b3
This commit is contained in:
parent
96f2694b67
commit
14abdf44c0
|
@ -1307,7 +1307,7 @@ class ZuulTestCase(BaseTestCase):
|
|||
# processed
|
||||
self.eventQueuesJoin()
|
||||
self.sched.run_handler_lock.acquire()
|
||||
if (not self.merge_client.build_sets and
|
||||
if (not self.merge_client.jobs and
|
||||
all(self.eventQueuesEmpty()) and
|
||||
self.haveAllBuildsReported() and
|
||||
self.areAllBuildsWaiting()):
|
||||
|
@ -1376,3 +1376,19 @@ tenants:
|
|||
""" % os.path.abspath(path))
|
||||
f.close()
|
||||
self.config.set('zuul', 'tenant_config', f.name)
|
||||
|
||||
def addCommitToRepo(self, project, message, files, branch='master'):
|
||||
path = os.path.join(self.upstream_root, project)
|
||||
repo = git.Repo(path)
|
||||
repo.head.reference = branch
|
||||
zuul.merger.merger.reset_repo_to_head(repo)
|
||||
for fn, content in files.items():
|
||||
fn = os.path.join(path, fn)
|
||||
with open(fn, 'w') as f:
|
||||
f.write(content)
|
||||
repo.index.add([fn])
|
||||
commit = repo.index.commit(message)
|
||||
repo.heads[branch].commit = commit
|
||||
repo.head.reference = branch
|
||||
repo.git.clean('-x', '-f', '-d')
|
||||
repo.heads[branch].checkout()
|
||||
|
|
|
@ -0,0 +1,36 @@
|
|||
pipelines:
|
||||
- name: check
|
||||
manager: IndependentPipelineManager
|
||||
source:
|
||||
gerrit
|
||||
trigger:
|
||||
gerrit:
|
||||
- event: patchset-created
|
||||
success:
|
||||
gerrit:
|
||||
verified: 1
|
||||
failure:
|
||||
gerrit:
|
||||
verified: -1
|
||||
|
||||
- name: tenant-one-gate
|
||||
manager: DependentPipelineManager
|
||||
success-message: Build succeeded (tenant-one-gate).
|
||||
source:
|
||||
gerrit
|
||||
trigger:
|
||||
gerrit:
|
||||
- event: comment-added
|
||||
approval:
|
||||
- approved: 1
|
||||
success:
|
||||
gerrit:
|
||||
verified: 2
|
||||
submit: true
|
||||
failure:
|
||||
gerrit:
|
||||
verified: -2
|
||||
start:
|
||||
gerrit:
|
||||
verified: 0
|
||||
precedence: high
|
|
@ -0,0 +1,8 @@
|
|||
tenants:
|
||||
- name: tenant-one
|
||||
include:
|
||||
- common.yaml
|
||||
source:
|
||||
gerrit:
|
||||
repos:
|
||||
- org/project
|
|
@ -0,0 +1,36 @@
|
|||
[gearman]
|
||||
server=127.0.0.1
|
||||
|
||||
[zuul]
|
||||
tenant_config=tests/fixtures/config/in-repo/main.yaml
|
||||
url_pattern=http://logs.example.com/{change.number}/{change.patchset}/{pipeline.name}/{job.name}/{build.number}
|
||||
job_name_in_report=true
|
||||
|
||||
[merger]
|
||||
git_dir=/tmp/zuul-test/git
|
||||
git_user_email=zuul@example.com
|
||||
git_user_name=zuul
|
||||
zuul_url=http://zuul.example.com/p
|
||||
|
||||
[swift]
|
||||
authurl=https://identity.api.example.org/v2.0/
|
||||
user=username
|
||||
key=password
|
||||
tenant_name=" "
|
||||
|
||||
default_container=logs
|
||||
region_name=EXP
|
||||
logserver_prefix=http://logs.example.org/server.app/
|
||||
|
||||
[connection gerrit]
|
||||
driver=gerrit
|
||||
server=review.example.com
|
||||
user=jenkins
|
||||
sshkey=none
|
||||
|
||||
[connection smtp]
|
||||
driver=smtp
|
||||
server=localhost
|
||||
port=25
|
||||
default_from=zuul@example.com
|
||||
default_to=you@example.com
|
|
@ -15,6 +15,7 @@
|
|||
# under the License.
|
||||
|
||||
import logging
|
||||
import textwrap
|
||||
|
||||
from tests.base import (
|
||||
ZuulTestCase,
|
||||
|
@ -62,3 +63,30 @@ class TestV3(ZuulTestCase):
|
|||
|
||||
self.assertEqual(A.reported, 2, "Activity in tenant two should"
|
||||
"not affect tenant one")
|
||||
|
||||
def test_in_repo_config(self):
|
||||
in_repo_conf = textwrap.dedent(
|
||||
"""
|
||||
projects:
|
||||
- name: org/project
|
||||
tenant-one-gate:
|
||||
- project-test1
|
||||
""")
|
||||
|
||||
self.addCommitToRepo('org/project', 'add zuul conf',
|
||||
{'.zuul.yaml': in_repo_conf})
|
||||
|
||||
self.setup_config('config/in-repo/zuul.conf')
|
||||
self.sched.reconfigure(self.config)
|
||||
|
||||
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
|
||||
A.addApproval('CRVW', 2)
|
||||
self.fake_gerrit.addEvent(A.addApproval('APRV', 1))
|
||||
self.waitUntilSettled()
|
||||
self.assertEqual(self.getJobFromHistory('project-test1').result,
|
||||
'SUCCESS')
|
||||
self.assertEqual(A.data['status'], 'MERGED')
|
||||
self.assertEqual(A.reported, 2,
|
||||
"A should report start and success")
|
||||
self.assertIn('tenant-one-gate', A.messages[1],
|
||||
"A should transit tenant-one gate")
|
||||
|
|
|
@ -37,7 +37,7 @@ class ConfigSchema(object):
|
|||
|
||||
def validateTenantSource(self, value, path=[]):
|
||||
# TODOv3(jeblair): validate against connections
|
||||
self.tenant_source.schema(value)
|
||||
self.tenant_source(value)
|
||||
|
||||
def getSchema(self, data, connections=None):
|
||||
tenant = {v.Required('name'): str,
|
||||
|
|
|
@ -14,6 +14,7 @@
|
|||
|
||||
import json
|
||||
import logging
|
||||
import threading
|
||||
from uuid import uuid4
|
||||
|
||||
import gear
|
||||
|
@ -55,6 +56,18 @@ class MergeGearmanClient(gear.Client):
|
|||
self.__merge_client.onBuildCompleted(job)
|
||||
|
||||
|
||||
class MergeJob(gear.Job):
|
||||
def __init__(self, *args, **kw):
|
||||
super(MergeJob, self).__init__(*args, **kw)
|
||||
self.__event = threading.Event()
|
||||
|
||||
def setComplete(self):
|
||||
self.__event.set()
|
||||
|
||||
def wait(self, timeout=300):
|
||||
return self.__event.wait(timeout)
|
||||
|
||||
|
||||
class MergeClient(object):
|
||||
log = logging.getLogger("zuul.MergeClient")
|
||||
|
||||
|
@ -71,26 +84,28 @@ class MergeClient(object):
|
|||
self.gearman.addServer(server, port)
|
||||
self.log.debug("Waiting for gearman")
|
||||
self.gearman.waitForServer()
|
||||
self.build_sets = {}
|
||||
self.jobs = set()
|
||||
|
||||
def stop(self):
|
||||
self.gearman.shutdown()
|
||||
|
||||
def areMergesOutstanding(self):
|
||||
if self.build_sets:
|
||||
if self.jobs:
|
||||
return True
|
||||
return False
|
||||
|
||||
def submitJob(self, name, data, build_set,
|
||||
precedence=zuul.model.PRECEDENCE_NORMAL):
|
||||
uuid = str(uuid4().hex)
|
||||
job = gear.Job(name,
|
||||
job = MergeJob(name,
|
||||
json.dumps(data),
|
||||
unique=uuid)
|
||||
job.build_set = build_set
|
||||
self.log.debug("Submitting job %s with data %s" % (job, data))
|
||||
self.build_sets[uuid] = build_set
|
||||
self.jobs.add(job)
|
||||
self.gearman.submitJob(job, precedence=precedence,
|
||||
timeout=300)
|
||||
return job
|
||||
|
||||
def mergeChanges(self, items, build_set,
|
||||
precedence=zuul.model.PRECEDENCE_NORMAL):
|
||||
|
@ -103,21 +118,29 @@ class MergeClient(object):
|
|||
url=url)
|
||||
self.submitJob('merger:update', data, build_set, precedence)
|
||||
|
||||
def getFiles(self, project, url, branch, files,
|
||||
precedence=zuul.model.PRECEDENCE_HIGH):
|
||||
data = dict(project=project,
|
||||
url=url,
|
||||
branch=branch,
|
||||
files=files)
|
||||
job = self.submitJob('merger:cat', data, None, precedence)
|
||||
return job
|
||||
|
||||
def onBuildCompleted(self, job):
|
||||
build_set = self.build_sets.get(job.unique)
|
||||
if build_set:
|
||||
data = getJobData(job)
|
||||
zuul_url = data.get('zuul_url')
|
||||
merged = data.get('merged', False)
|
||||
updated = data.get('updated', False)
|
||||
commit = data.get('commit')
|
||||
self.log.info("Merge %s complete, merged: %s, updated: %s, "
|
||||
"commit: %s" %
|
||||
(job, merged, updated, build_set.commit))
|
||||
self.sched.onMergeCompleted(build_set, zuul_url,
|
||||
data = getJobData(job)
|
||||
zuul_url = data.get('zuul_url')
|
||||
merged = data.get('merged', False)
|
||||
updated = data.get('updated', False)
|
||||
commit = data.get('commit')
|
||||
job.files = data.get('files', {})
|
||||
self.log.info("Merge %s complete, merged: %s, updated: %s, "
|
||||
"commit: %s" %
|
||||
(job, merged, updated, commit))
|
||||
job.setComplete()
|
||||
if job.build_set:
|
||||
self.sched.onMergeCompleted(job.build_set, zuul_url,
|
||||
merged, updated, commit)
|
||||
# The test suite expects the build_set to be removed from
|
||||
# the internal dict after the wake flag is set.
|
||||
del self.build_sets[job.unique]
|
||||
else:
|
||||
self.log.error("Unable to find build set for uuid %s" % job.unique)
|
||||
# The test suite expects the job to be removed from the
|
||||
# internal account after the wake flag is set.
|
||||
self.jobs.remove(job)
|
||||
|
|
|
@ -184,6 +184,17 @@ class Repo(object):
|
|||
origin = repo.remotes.origin
|
||||
origin.update()
|
||||
|
||||
def getFiles(self, branch, files):
|
||||
ret = {}
|
||||
repo = self.createRepoObject()
|
||||
for fn in files:
|
||||
tree = repo.heads[branch].commit.tree
|
||||
if fn in tree:
|
||||
ret[fn] = tree[fn].data_stream.read()
|
||||
else:
|
||||
ret[fn] = None
|
||||
return ret
|
||||
|
||||
|
||||
class Merger(object):
|
||||
log = logging.getLogger("zuul.Merger")
|
||||
|
@ -342,3 +353,7 @@ class Merger(object):
|
|||
if not commit:
|
||||
return None
|
||||
return commit.hexsha
|
||||
|
||||
def getFiles(self, project, url, branch, files):
|
||||
repo = self.getRepo(project, url)
|
||||
return repo.getFiles(branch, files)
|
||||
|
|
|
@ -68,6 +68,7 @@ class MergeServer(object):
|
|||
def register(self):
|
||||
self.worker.registerFunction("merger:merge")
|
||||
self.worker.registerFunction("merger:update")
|
||||
self.worker.registerFunction("merger:cat")
|
||||
|
||||
def stop(self):
|
||||
self.log.debug("Stopping")
|
||||
|
@ -90,6 +91,9 @@ class MergeServer(object):
|
|||
elif job.name == 'merger:update':
|
||||
self.log.debug("Got update job: %s" % job.unique)
|
||||
self.update(job)
|
||||
elif job.name == 'merger:cat':
|
||||
self.log.debug("Got cat job: %s" % job.unique)
|
||||
self.cat(job)
|
||||
else:
|
||||
self.log.error("Unable to handle job %s" % job.name)
|
||||
job.sendWorkFail()
|
||||
|
@ -113,3 +117,13 @@ class MergeServer(object):
|
|||
result = dict(updated=True,
|
||||
zuul_url=self.zuul_url)
|
||||
job.sendWorkComplete(json.dumps(result))
|
||||
|
||||
def cat(self, job):
|
||||
args = json.loads(job.arguments)
|
||||
self.merger.updateRepo(args['project'], args['url'])
|
||||
files = self.merger.getFiles(args['project'], args['url'],
|
||||
args['branch'], args['files'])
|
||||
result = dict(updated=True,
|
||||
files=files,
|
||||
zuul_url=self.zuul_url)
|
||||
job.sendWorkComplete(json.dumps(result))
|
||||
|
|
|
@ -353,6 +353,7 @@ class Scheduler(threading.Thread):
|
|||
raise Exception("Unable to read tenant config file at %s" %
|
||||
config_path)
|
||||
with open(config_path) as config_file:
|
||||
self.log.info("Loading configuration from %s" % (config_path,))
|
||||
data = yaml.load(config_file)
|
||||
base = os.path.dirname(os.path.realpath(config_path))
|
||||
|
||||
|
@ -368,8 +369,11 @@ class Scheduler(threading.Thread):
|
|||
fn = os.path.join(base, fn)
|
||||
fn = os.path.expanduser(fn)
|
||||
with open(fn) as config_file:
|
||||
self.log.info("Loading configuration from %s" % (fn,))
|
||||
incdata = yaml.load(config_file)
|
||||
extend_dict(tenant_config, incdata)
|
||||
incdata = self._parseTenantInRepoLayouts(conf_tenant)
|
||||
extend_dict(tenant_config, incdata)
|
||||
tenant.layout = self._parseLayout(base, tenant_config, connections)
|
||||
return abide
|
||||
|
||||
|
@ -583,6 +587,39 @@ class Scheduler(threading.Thread):
|
|||
|
||||
return layout
|
||||
|
||||
def _parseTenantInRepoLayouts(self, conf_tenant):
|
||||
config = {}
|
||||
jobs = []
|
||||
for source_name, conf_source in conf_tenant.get('source', {}).items():
|
||||
# TODOv3(jeblair,jhesketh): sources should just be
|
||||
# set up at the start of the zuul.conf parsing
|
||||
if source_name not in self.sources:
|
||||
self.sources[source_name] = self._getSourceDriver(
|
||||
source_name)
|
||||
for conf_repo in conf_source.get('repos'):
|
||||
source = self.sources[source_name]
|
||||
project = source.getProject(conf_repo)
|
||||
url = source.getGitUrl(project)
|
||||
# TODOv3(jeblair): config should be branch specific
|
||||
job = self.merger.getFiles(project.name, url, 'master',
|
||||
files=['.zuul.yaml'])
|
||||
job.project = project
|
||||
jobs.append(job)
|
||||
for job in jobs:
|
||||
self.log.debug("Waiting for cat job %s" % (job,))
|
||||
job.wait()
|
||||
if job.files.get('.zuul.yaml'):
|
||||
self.log.info("Loading configuration from %s/.zuul.yaml" %
|
||||
(job.project,))
|
||||
incdata = self._parseInRepoLayout(job.files['.zuul.yaml'])
|
||||
extend_dict(config, incdata)
|
||||
return config
|
||||
|
||||
def _parseInRepoLayout(self, data):
|
||||
# TODOv3(jeblair): this should implement some rules to protect
|
||||
# aspects of the config that should not be changed in-repo
|
||||
return yaml.load(data)
|
||||
|
||||
def setLauncher(self, launcher):
|
||||
self.launcher = launcher
|
||||
|
||||
|
|
Loading…
Reference in New Issue