diff --git a/tests/base.py b/tests/base.py index 5e1befe37d..497d706883 100755 --- a/tests/base.py +++ b/tests/base.py @@ -1307,7 +1307,7 @@ class ZuulTestCase(BaseTestCase): # processed self.eventQueuesJoin() self.sched.run_handler_lock.acquire() - if (not self.merge_client.build_sets and + if (not self.merge_client.jobs and all(self.eventQueuesEmpty()) and self.haveAllBuildsReported() and self.areAllBuildsWaiting()): @@ -1376,3 +1376,19 @@ tenants: """ % os.path.abspath(path)) f.close() self.config.set('zuul', 'tenant_config', f.name) + + def addCommitToRepo(self, project, message, files, branch='master'): + path = os.path.join(self.upstream_root, project) + repo = git.Repo(path) + repo.head.reference = branch + zuul.merger.merger.reset_repo_to_head(repo) + for fn, content in files.items(): + fn = os.path.join(path, fn) + with open(fn, 'w') as f: + f.write(content) + repo.index.add([fn]) + commit = repo.index.commit(message) + repo.heads[branch].commit = commit + repo.head.reference = branch + repo.git.clean('-x', '-f', '-d') + repo.heads[branch].checkout() diff --git a/tests/fixtures/config/in-repo/common.yaml b/tests/fixtures/config/in-repo/common.yaml new file mode 100644 index 0000000000..96aebd6316 --- /dev/null +++ b/tests/fixtures/config/in-repo/common.yaml @@ -0,0 +1,36 @@ +pipelines: + - name: check + manager: IndependentPipelineManager + source: + gerrit + trigger: + gerrit: + - event: patchset-created + success: + gerrit: + verified: 1 + failure: + gerrit: + verified: -1 + + - name: tenant-one-gate + manager: DependentPipelineManager + success-message: Build succeeded (tenant-one-gate). + source: + gerrit + trigger: + gerrit: + - event: comment-added + approval: + - approved: 1 + success: + gerrit: + verified: 2 + submit: true + failure: + gerrit: + verified: -2 + start: + gerrit: + verified: 0 + precedence: high diff --git a/tests/fixtures/config/in-repo/main.yaml b/tests/fixtures/config/in-repo/main.yaml new file mode 100644 index 0000000000..df7daddbef --- /dev/null +++ b/tests/fixtures/config/in-repo/main.yaml @@ -0,0 +1,8 @@ +tenants: + - name: tenant-one + include: + - common.yaml + source: + gerrit: + repos: + - org/project diff --git a/tests/fixtures/config/in-repo/zuul.conf b/tests/fixtures/config/in-repo/zuul.conf new file mode 100644 index 0000000000..14708aa791 --- /dev/null +++ b/tests/fixtures/config/in-repo/zuul.conf @@ -0,0 +1,36 @@ +[gearman] +server=127.0.0.1 + +[zuul] +tenant_config=tests/fixtures/config/in-repo/main.yaml +url_pattern=http://logs.example.com/{change.number}/{change.patchset}/{pipeline.name}/{job.name}/{build.number} +job_name_in_report=true + +[merger] +git_dir=/tmp/zuul-test/git +git_user_email=zuul@example.com +git_user_name=zuul +zuul_url=http://zuul.example.com/p + +[swift] +authurl=https://identity.api.example.org/v2.0/ +user=username +key=password +tenant_name=" " + +default_container=logs +region_name=EXP +logserver_prefix=http://logs.example.org/server.app/ + +[connection gerrit] +driver=gerrit +server=review.example.com +user=jenkins +sshkey=none + +[connection smtp] +driver=smtp +server=localhost +port=25 +default_from=zuul@example.com +default_to=you@example.com diff --git a/tests/test_v3.py b/tests/test_v3.py index 2e1674203b..69e66a0113 100644 --- a/tests/test_v3.py +++ b/tests/test_v3.py @@ -15,6 +15,7 @@ # under the License. import logging +import textwrap from tests.base import ( ZuulTestCase, @@ -62,3 +63,30 @@ class TestV3(ZuulTestCase): self.assertEqual(A.reported, 2, "Activity in tenant two should" "not affect tenant one") + + def test_in_repo_config(self): + in_repo_conf = textwrap.dedent( + """ + projects: + - name: org/project + tenant-one-gate: + - project-test1 + """) + + self.addCommitToRepo('org/project', 'add zuul conf', + {'.zuul.yaml': in_repo_conf}) + + self.setup_config('config/in-repo/zuul.conf') + self.sched.reconfigure(self.config) + + A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') + A.addApproval('CRVW', 2) + self.fake_gerrit.addEvent(A.addApproval('APRV', 1)) + self.waitUntilSettled() + self.assertEqual(self.getJobFromHistory('project-test1').result, + 'SUCCESS') + self.assertEqual(A.data['status'], 'MERGED') + self.assertEqual(A.reported, 2, + "A should report start and success") + self.assertIn('tenant-one-gate', A.messages[1], + "A should transit tenant-one gate") diff --git a/zuul/layoutvalidator.py b/zuul/layoutvalidator.py index e7bddba856..0adb78cdae 100644 --- a/zuul/layoutvalidator.py +++ b/zuul/layoutvalidator.py @@ -37,7 +37,7 @@ class ConfigSchema(object): def validateTenantSource(self, value, path=[]): # TODOv3(jeblair): validate against connections - self.tenant_source.schema(value) + self.tenant_source(value) def getSchema(self, data, connections=None): tenant = {v.Required('name'): str, diff --git a/zuul/merger/client.py b/zuul/merger/client.py index 950c385d3a..ce04795bfa 100644 --- a/zuul/merger/client.py +++ b/zuul/merger/client.py @@ -14,6 +14,7 @@ import json import logging +import threading from uuid import uuid4 import gear @@ -55,6 +56,18 @@ class MergeGearmanClient(gear.Client): self.__merge_client.onBuildCompleted(job) +class MergeJob(gear.Job): + def __init__(self, *args, **kw): + super(MergeJob, self).__init__(*args, **kw) + self.__event = threading.Event() + + def setComplete(self): + self.__event.set() + + def wait(self, timeout=300): + return self.__event.wait(timeout) + + class MergeClient(object): log = logging.getLogger("zuul.MergeClient") @@ -71,26 +84,28 @@ class MergeClient(object): self.gearman.addServer(server, port) self.log.debug("Waiting for gearman") self.gearman.waitForServer() - self.build_sets = {} + self.jobs = set() def stop(self): self.gearman.shutdown() def areMergesOutstanding(self): - if self.build_sets: + if self.jobs: return True return False def submitJob(self, name, data, build_set, precedence=zuul.model.PRECEDENCE_NORMAL): uuid = str(uuid4().hex) - job = gear.Job(name, + job = MergeJob(name, json.dumps(data), unique=uuid) + job.build_set = build_set self.log.debug("Submitting job %s with data %s" % (job, data)) - self.build_sets[uuid] = build_set + self.jobs.add(job) self.gearman.submitJob(job, precedence=precedence, timeout=300) + return job def mergeChanges(self, items, build_set, precedence=zuul.model.PRECEDENCE_NORMAL): @@ -103,21 +118,29 @@ class MergeClient(object): url=url) self.submitJob('merger:update', data, build_set, precedence) + def getFiles(self, project, url, branch, files, + precedence=zuul.model.PRECEDENCE_HIGH): + data = dict(project=project, + url=url, + branch=branch, + files=files) + job = self.submitJob('merger:cat', data, None, precedence) + return job + def onBuildCompleted(self, job): - build_set = self.build_sets.get(job.unique) - if build_set: - data = getJobData(job) - zuul_url = data.get('zuul_url') - merged = data.get('merged', False) - updated = data.get('updated', False) - commit = data.get('commit') - self.log.info("Merge %s complete, merged: %s, updated: %s, " - "commit: %s" % - (job, merged, updated, build_set.commit)) - self.sched.onMergeCompleted(build_set, zuul_url, + data = getJobData(job) + zuul_url = data.get('zuul_url') + merged = data.get('merged', False) + updated = data.get('updated', False) + commit = data.get('commit') + job.files = data.get('files', {}) + self.log.info("Merge %s complete, merged: %s, updated: %s, " + "commit: %s" % + (job, merged, updated, commit)) + job.setComplete() + if job.build_set: + self.sched.onMergeCompleted(job.build_set, zuul_url, merged, updated, commit) - # The test suite expects the build_set to be removed from - # the internal dict after the wake flag is set. - del self.build_sets[job.unique] - else: - self.log.error("Unable to find build set for uuid %s" % job.unique) + # The test suite expects the job to be removed from the + # internal account after the wake flag is set. + self.jobs.remove(job) diff --git a/zuul/merger/merger.py b/zuul/merger/merger.py index c6ae35d79a..b7e1842d38 100644 --- a/zuul/merger/merger.py +++ b/zuul/merger/merger.py @@ -184,6 +184,17 @@ class Repo(object): origin = repo.remotes.origin origin.update() + def getFiles(self, branch, files): + ret = {} + repo = self.createRepoObject() + for fn in files: + tree = repo.heads[branch].commit.tree + if fn in tree: + ret[fn] = tree[fn].data_stream.read() + else: + ret[fn] = None + return ret + class Merger(object): log = logging.getLogger("zuul.Merger") @@ -342,3 +353,7 @@ class Merger(object): if not commit: return None return commit.hexsha + + def getFiles(self, project, url, branch, files): + repo = self.getRepo(project, url) + return repo.getFiles(branch, files) diff --git a/zuul/merger/server.py b/zuul/merger/server.py index 30cd732ecb..813c60241c 100644 --- a/zuul/merger/server.py +++ b/zuul/merger/server.py @@ -68,6 +68,7 @@ class MergeServer(object): def register(self): self.worker.registerFunction("merger:merge") self.worker.registerFunction("merger:update") + self.worker.registerFunction("merger:cat") def stop(self): self.log.debug("Stopping") @@ -90,6 +91,9 @@ class MergeServer(object): elif job.name == 'merger:update': self.log.debug("Got update job: %s" % job.unique) self.update(job) + elif job.name == 'merger:cat': + self.log.debug("Got cat job: %s" % job.unique) + self.cat(job) else: self.log.error("Unable to handle job %s" % job.name) job.sendWorkFail() @@ -113,3 +117,13 @@ class MergeServer(object): result = dict(updated=True, zuul_url=self.zuul_url) job.sendWorkComplete(json.dumps(result)) + + def cat(self, job): + args = json.loads(job.arguments) + self.merger.updateRepo(args['project'], args['url']) + files = self.merger.getFiles(args['project'], args['url'], + args['branch'], args['files']) + result = dict(updated=True, + files=files, + zuul_url=self.zuul_url) + job.sendWorkComplete(json.dumps(result)) diff --git a/zuul/scheduler.py b/zuul/scheduler.py index 06c54cf613..5ae9e9b8cc 100644 --- a/zuul/scheduler.py +++ b/zuul/scheduler.py @@ -353,6 +353,7 @@ class Scheduler(threading.Thread): raise Exception("Unable to read tenant config file at %s" % config_path) with open(config_path) as config_file: + self.log.info("Loading configuration from %s" % (config_path,)) data = yaml.load(config_file) base = os.path.dirname(os.path.realpath(config_path)) @@ -368,8 +369,11 @@ class Scheduler(threading.Thread): fn = os.path.join(base, fn) fn = os.path.expanduser(fn) with open(fn) as config_file: + self.log.info("Loading configuration from %s" % (fn,)) incdata = yaml.load(config_file) extend_dict(tenant_config, incdata) + incdata = self._parseTenantInRepoLayouts(conf_tenant) + extend_dict(tenant_config, incdata) tenant.layout = self._parseLayout(base, tenant_config, connections) return abide @@ -583,6 +587,39 @@ class Scheduler(threading.Thread): return layout + def _parseTenantInRepoLayouts(self, conf_tenant): + config = {} + jobs = [] + for source_name, conf_source in conf_tenant.get('source', {}).items(): + # TODOv3(jeblair,jhesketh): sources should just be + # set up at the start of the zuul.conf parsing + if source_name not in self.sources: + self.sources[source_name] = self._getSourceDriver( + source_name) + for conf_repo in conf_source.get('repos'): + source = self.sources[source_name] + project = source.getProject(conf_repo) + url = source.getGitUrl(project) + # TODOv3(jeblair): config should be branch specific + job = self.merger.getFiles(project.name, url, 'master', + files=['.zuul.yaml']) + job.project = project + jobs.append(job) + for job in jobs: + self.log.debug("Waiting for cat job %s" % (job,)) + job.wait() + if job.files.get('.zuul.yaml'): + self.log.info("Loading configuration from %s/.zuul.yaml" % + (job.project,)) + incdata = self._parseInRepoLayout(job.files['.zuul.yaml']) + extend_dict(config, incdata) + return config + + def _parseInRepoLayout(self, data): + # TODOv3(jeblair): this should implement some rules to protect + # aspects of the config that should not be changed in-repo + return yaml.load(data) + def setLauncher(self, launcher): self.launcher = launcher