diff --git a/tests/base.py b/tests/base.py index b265bd0361..736d2f792e 100755 --- a/tests/base.py +++ b/tests/base.py @@ -974,10 +974,6 @@ class ZuulTestCase(BaseTestCase): old_urlopen = urllib2.urlopen urllib2.urlopen = URLOpenerFactory - self.merge_server = zuul.merger.server.MergeServer(self.config, - self.connections) - self.merge_server.start() - self.launcher = zuul.launcher.launchclient.LaunchClient( self.config, self.sched, self.swift) self.merge_client = zuul.merger.client.MergeClient( @@ -1076,6 +1072,32 @@ class ZuulTestCase(BaseTestCase): self.config.read(os.path.join(FIXTURE_DIR, self.config_file)) if hasattr(self, 'tenant_config_file'): self.config.set('zuul', 'tenant_config', self.tenant_config_file) + git_path = os.path.join( + os.path.dirname( + os.path.join(FIXTURE_DIR, self.tenant_config_file)), + 'git') + if os.path.exists(git_path): + for reponame in os.listdir(git_path): + self.copyDirToRepo(reponame, + os.path.join(git_path, reponame)) + + def copyDirToRepo(self, project, source_path): + repo_path = os.path.join(self.upstream_root, project) + if not os.path.exists(repo_path): + self.init_repo(project) + + files = {} + for (dirpath, dirnames, filenames) in os.walk(source_path): + for filename in filenames: + test_tree_filepath = os.path.join(dirpath, filename) + common_path = os.path.commonprefix([test_tree_filepath, + source_path]) + relative_filepath = test_tree_filepath[len(common_path) + 1:] + with open(test_tree_filepath, 'r') as f: + content = f.read() + files[relative_filepath] = content + self.addCommitToRepo(project, 'add content from fixture', + files, branch='master') def setup_repos(self): """Subclasses can override to manipulate repos before tests""" @@ -1099,8 +1121,6 @@ class ZuulTestCase(BaseTestCase): def shutdown(self): self.log.debug("Shutting down after tests") self.launcher.stop() - self.merge_server.stop() - self.merge_server.join() self.merge_client.stop() self.sched.stop() self.sched.join() @@ -1233,7 +1253,6 @@ class ZuulTestCase(BaseTestCase): time.sleep(0) self.gearman_server.functions = set() self.rpc.register() - self.merge_server.register() def haveAllBuildsReported(self): # See if Zuul is waiting on a meta job to complete @@ -1331,13 +1350,13 @@ class ZuulTestCase(BaseTestCase): jobs = filter(lambda x: x.result == result, jobs) return len(jobs) - def getJobFromHistory(self, name): + def getJobFromHistory(self, name, project=None): history = self.ansible_server.job_history for job in history: params = json.loads(job.arguments) - if params['job'] == name: + if (params['job'] == name and + (project is None or params['ZUUL_PROJECT'] == project)): result = json.loads(job.data[-1]) - print result ret = BuildHistory(job=job, name=params['job'], result=result['result']) diff --git a/tests/fixtures/config/in-repo/common.yaml b/tests/fixtures/config/in-repo/git/common-config/zuul.yaml similarity index 100% rename from tests/fixtures/config/in-repo/common.yaml rename to tests/fixtures/config/in-repo/git/common-config/zuul.yaml diff --git a/tests/fixtures/config/in-repo/main.yaml b/tests/fixtures/config/in-repo/main.yaml index e8b7665c35..d9868fad0f 100644 --- a/tests/fixtures/config/in-repo/main.yaml +++ b/tests/fixtures/config/in-repo/main.yaml @@ -1,8 +1,8 @@ - tenant: name: tenant-one - include: - - common.yaml source: gerrit: - repos: + config-repos: + - common-config + project-repos: - org/project diff --git a/tests/fixtures/config/multi-tenant/common.yaml b/tests/fixtures/config/multi-tenant/git/common-config/zuul.yaml similarity index 87% rename from tests/fixtures/config/multi-tenant/common.yaml rename to tests/fixtures/config/multi-tenant/git/common-config/zuul.yaml index 60142273aa..77195731bf 100644 --- a/tests/fixtures/config/multi-tenant/common.yaml +++ b/tests/fixtures/config/multi-tenant/git/common-config/zuul.yaml @@ -12,3 +12,7 @@ failure: gerrit: verified: -1 + +- job: + name: + python27 diff --git a/tests/fixtures/config/multi-tenant/tenant-one.yaml b/tests/fixtures/config/multi-tenant/git/tenant-one-config/zuul.yaml similarity index 94% rename from tests/fixtures/config/multi-tenant/tenant-one.yaml rename to tests/fixtures/config/multi-tenant/git/tenant-one-config/zuul.yaml index 86a98da641..785f8a53c4 100644 --- a/tests/fixtures/config/multi-tenant/tenant-one.yaml +++ b/tests/fixtures/config/multi-tenant/git/tenant-one-config/zuul.yaml @@ -29,7 +29,9 @@ name: org/project1 check: jobs: + - python27 - project1-test1 tenant-one-gate: jobs: + - python27 - project1-test1 diff --git a/tests/fixtures/config/multi-tenant/tenant-two.yaml b/tests/fixtures/config/multi-tenant/git/tenant-two-config/zuul.yaml similarity index 94% rename from tests/fixtures/config/multi-tenant/tenant-two.yaml rename to tests/fixtures/config/multi-tenant/git/tenant-two-config/zuul.yaml index 3f80a95597..c6127ca968 100644 --- a/tests/fixtures/config/multi-tenant/tenant-two.yaml +++ b/tests/fixtures/config/multi-tenant/git/tenant-two-config/zuul.yaml @@ -29,7 +29,9 @@ name: org/project2 check: jobs: + - python27 - project2-test1 tenant-two-gate: jobs: + - python27 - project2-test1 diff --git a/tests/fixtures/config/multi-tenant/main.yaml b/tests/fixtures/config/multi-tenant/main.yaml index b9d780cee4..b1c47b1154 100644 --- a/tests/fixtures/config/multi-tenant/main.yaml +++ b/tests/fixtures/config/multi-tenant/main.yaml @@ -1,11 +1,15 @@ - tenant: name: tenant-one - include: - - common.yaml - - tenant-one.yaml + source: + gerrit: + config-repos: + - common-config + - tenant-one-config - tenant: name: tenant-two - include: - - common.yaml - - tenant-two.yaml + source: + gerrit: + config-repos: + - common-config + - tenant-two-config diff --git a/tests/fixtures/config/project-template/common.yaml b/tests/fixtures/config/project-template/git/common-config/zuul.yaml similarity index 100% rename from tests/fixtures/config/project-template/common.yaml rename to tests/fixtures/config/project-template/git/common-config/zuul.yaml diff --git a/tests/fixtures/config/project-template/main.yaml b/tests/fixtures/config/project-template/main.yaml index 25dea572e4..a22ed5c60c 100644 --- a/tests/fixtures/config/project-template/main.yaml +++ b/tests/fixtures/config/project-template/main.yaml @@ -1,4 +1,6 @@ - tenant: name: tenant-one - include: - - common.yaml + source: + gerrit: + config-repos: + - common-config diff --git a/tests/test_v3.py b/tests/test_v3.py index 8425383c15..8874015bdf 100644 --- a/tests/test_v3.py +++ b/tests/test_v3.py @@ -38,6 +38,8 @@ class TestMultipleTenants(ZuulTestCase): self.waitUntilSettled() self.assertEqual(self.getJobFromHistory('project1-test1').result, 'SUCCESS') + self.assertEqual(self.getJobFromHistory('python27').result, + 'SUCCESS') self.assertEqual(A.data['status'], 'MERGED') self.assertEqual(A.reported, 2, "A should report start and success") @@ -50,6 +52,9 @@ class TestMultipleTenants(ZuulTestCase): B.addApproval('CRVW', 2) self.fake_gerrit.addEvent(B.addApproval('APRV', 1)) self.waitUntilSettled() + self.assertEqual(self.getJobFromHistory('python27', + 'org/project2').result, + 'SUCCESS') self.assertEqual(self.getJobFromHistory('project2-test1').result, 'SUCCESS') self.assertEqual(B.data['status'], 'MERGED') diff --git a/zuul/configloader.py b/zuul/configloader.py index 26db6ed0d4..1703b0f853 100644 --- a/zuul/configloader.py +++ b/zuul/configloader.py @@ -67,6 +67,8 @@ class JobParser(object): 'swift': to_list(swift), 'irrelevant-files': to_list(str), 'timeout': int, + '_project_source': str, # used internally + '_project_name': str, # used internally } return vs.Schema(job) @@ -91,11 +93,17 @@ class JobParser(object): # accumulate onto any previously applied tags from # metajobs. job.tags = job.tags.union(set(tags)) - + if not job.project_source: + # Thes attributes may not be overidden -- the first + # reference definition of a job is in the repo where it is + # first defined. + job.project_source = conf.get('_project_source') + job.project_name = conf.get('_project_name') job.failure_message = conf.get('failure-message', job.failure_message) job.success_message = conf.get('success-message', job.success_message) job.failure_url = conf.get('failure-url', job.failure_url) job.success_url = conf.get('success-url', job.success_url) + if 'branches' in conf: matchers = [] for branch in as_list(conf['branches']): @@ -413,7 +421,8 @@ class PipelineParser(object): class TenantParser(object): log = logging.getLogger("zuul.TenantParser") - tenant_source = vs.Schema({'repos': [str]}) + tenant_source = vs.Schema({'config-repos': [str], + 'project-repos': [str]}) @staticmethod def validateTenantSources(connections): @@ -433,7 +442,6 @@ class TenantParser(object): @staticmethod def getSchema(connections=None): tenant = {vs.Required('name'): str, - 'include': to_list(str), 'source': TenantParser.validateTenantSources(connections)} return vs.Schema(tenant) @@ -442,14 +450,6 @@ class TenantParser(object): TenantParser.getSchema(connections)(conf) tenant = model.Tenant(conf['name']) tenant_config = model.UnparsedTenantConfig() - for fn in conf.get('include', []): - if not os.path.isabs(fn): - fn = os.path.join(base, fn) - fn = os.path.expanduser(fn) - with open(fn) as config_file: - TenantParser.log.info("Loading configuration from %s" % (fn,)) - incdata = yaml.load(config_file) - tenant_config.extend(incdata) incdata = TenantParser._loadTenantInRepoLayouts(merger, connections, conf) tenant_config.extend(incdata) @@ -463,31 +463,77 @@ class TenantParser(object): jobs = [] for source_name, conf_source in conf_tenant.get('source', {}).items(): source = connections.getSource(source_name) - for conf_repo in conf_source.get('repos'): + + # Get main config files. These files are permitted the + # full range of configuration. + for conf_repo in conf_source.get('config-repos', []): + project = source.getProject(conf_repo) + url = source.getGitUrl(project) + job = merger.getFiles(project.name, url, 'master', + files=['zuul.yaml', '.zuul.yaml']) + job.project = project + job.config_repo = True + jobs.append(job) + + # Get in-project-repo config files which have a restricted + # set of options. + for conf_repo in conf_source.get('project-repos', []): project = source.getProject(conf_repo) url = source.getGitUrl(project) # TODOv3(jeblair): config should be branch specific job = merger.getFiles(project.name, url, 'master', files=['.zuul.yaml']) job.project = project + job.config_repo = False jobs.append(job) + for job in jobs: + # Note: this is an ordered list -- we wait for cat jobs to + # complete in the order they were launched which is the + # same order they were defined in the main config file. + # This is important for correct inheritence. TenantParser.log.debug("Waiting for cat job %s" % (job,)) job.wait() - if job.files.get('.zuul.yaml'): - TenantParser.log.info( - "Loading configuration from %s/.zuul.yaml" % - (job.project,)) - incdata = TenantParser._parseInRepoLayout( - job.files['.zuul.yaml']) - config.extend(incdata) + for fn in ['zuul.yaml', '.zuul.yaml']: + if job.files.get(fn): + TenantParser.log.info( + "Loading configuration from %s/%s" % + (job.project, fn)) + if job.config_repo: + incdata = TenantParser._parseConfigRepoLayout( + job.files[fn], source_name, job.project.name) + else: + incdata = TenantParser._parseProjectRepoLayout( + job.files[fn], source_name, job.project.name) + config.extend(incdata) return config @staticmethod - def _parseInRepoLayout(data): + def _parseConfigRepoLayout(data, source_name, project_name): + # This is the top-level configuration for a tenant. + config = model.UnparsedTenantConfig() + config.extend(yaml.load(data)) + + # Remember where this job was defined + for conf_job in config.jobs: + conf_job['_project_source'] = source_name + conf_job['_project_name'] = project_name + + return config + + @staticmethod + def _parseProjectRepoLayout(data, source_name, project_name): # TODOv3(jeblair): this should implement some rules to protect # aspects of the config that should not be changed in-repo - return yaml.load(data) + config = model.UnparsedTenantConfig() + config.extend(yaml.load(data)) + + # Remember where this job was defined + for conf_job in config.jobs: + conf_job['_project_source'] = source_name + conf_job['_project_name'] = project_name + + return config @staticmethod def _parseLayout(base, data, scheduler, connections): diff --git a/zuul/merger/merger.py b/zuul/merger/merger.py index eaa5721f72..7c9f1f9886 100644 --- a/zuul/merger/merger.py +++ b/zuul/merger/merger.py @@ -77,12 +77,8 @@ class Repo(object): return self._initialized def createRepoObject(self): - try: - self._ensure_cloned() - repo = git.Repo(self.local_path) - except: - self.log.exception("Unable to initialize repo for %s" % - self.local_path) + self._ensure_cloned() + repo = git.Repo(self.local_path) return repo def reset(self): diff --git a/zuul/model.py b/zuul/model.py index a88c365d41..93fbb3182d 100644 --- a/zuul/model.py +++ b/zuul/model.py @@ -455,6 +455,8 @@ class Job(object): pre_run=None, post_run=None, voting=None, + project_source=None, + project_name=None, failure_message=None, success_message=None, failure_url=None,