Add support for zuul.d configuration split
This change implements the zuul_split spec to support configuration split in a zuul.d directory. Change-Id: I6bc7250b2045b73dfba109aa0b2f1ba5d66752b2
This commit is contained in:
parent
2c414c1ca2
commit
829e617bac
2
tests/fixtures/config/split-config/git/common-config/playbooks/project-test1.yaml
vendored
Normal file
2
tests/fixtures/config/split-config/git/common-config/playbooks/project-test1.yaml
vendored
Normal file
|
@ -0,0 +1,2 @@
|
|||
- hosts: all
|
||||
tasks: []
|
|
@ -0,0 +1,2 @@
|
|||
- job:
|
||||
name: project-test1
|
|
@ -0,0 +1,5 @@
|
|||
- project:
|
||||
name: org/project
|
||||
check:
|
||||
jobs:
|
||||
- project-test1
|
|
@ -0,0 +1,12 @@
|
|||
- pipeline:
|
||||
name: check
|
||||
manager: independent
|
||||
trigger:
|
||||
gerrit:
|
||||
- event: patchset-created
|
||||
success:
|
||||
gerrit:
|
||||
verified: 1
|
||||
failure:
|
||||
gerrit:
|
||||
verified: -1
|
|
@ -0,0 +1 @@
|
|||
test
|
|
@ -0,0 +1,7 @@
|
|||
- project:
|
||||
name: org/project1
|
||||
check:
|
||||
jobs:
|
||||
- project-test1
|
||||
- project1-project2-integration:
|
||||
dependencies: project-test1
|
|
@ -0,0 +1,2 @@
|
|||
- job:
|
||||
name: project1-project2-integration
|
|
@ -0,0 +1 @@
|
|||
test
|
|
@ -0,0 +1,2 @@
|
|||
- hosts: all
|
||||
tasks: []
|
|
@ -0,0 +1,9 @@
|
|||
- tenant:
|
||||
name: tenant-one
|
||||
source:
|
||||
gerrit:
|
||||
config-projects:
|
||||
- common-config
|
||||
untrusted-projects:
|
||||
- org/project
|
||||
- org/project1
|
|
@ -0,0 +1,17 @@
|
|||
- job:
|
||||
name: project-test1
|
||||
|
||||
- job:
|
||||
name: project-test2
|
||||
|
||||
- job:
|
||||
name: layered-project-test3
|
||||
|
||||
- job:
|
||||
name: layered-project-test4
|
||||
|
||||
- job:
|
||||
name: layered-project-foo-test5
|
||||
|
||||
- job:
|
||||
name: project-test6
|
41
tests/fixtures/config/templated-project/git/common-config/zuul.d/pipelines.yaml
vendored
Normal file
41
tests/fixtures/config/templated-project/git/common-config/zuul.d/pipelines.yaml
vendored
Normal file
|
@ -0,0 +1,41 @@
|
|||
- pipeline:
|
||||
name: check
|
||||
manager: independent
|
||||
trigger:
|
||||
gerrit:
|
||||
- event: patchset-created
|
||||
success:
|
||||
gerrit:
|
||||
verified: 1
|
||||
failure:
|
||||
gerrit:
|
||||
verified: -1
|
||||
|
||||
- pipeline:
|
||||
name: gate
|
||||
manager: dependent
|
||||
success-message: Build succeeded (gate).
|
||||
trigger:
|
||||
gerrit:
|
||||
- event: comment-added
|
||||
approval:
|
||||
- approved: 1
|
||||
success:
|
||||
gerrit:
|
||||
verified: 2
|
||||
submit: true
|
||||
failure:
|
||||
gerrit:
|
||||
verified: -2
|
||||
start:
|
||||
gerrit:
|
||||
verified: 0
|
||||
precedence: high
|
||||
|
||||
- pipeline:
|
||||
name: post
|
||||
manager: independent
|
||||
trigger:
|
||||
gerrit:
|
||||
- event: ref-updated
|
||||
ref: ^(?!refs/).*$
|
14
tests/fixtures/config/templated-project/git/common-config/zuul.d/projects.yaml
vendored
Normal file
14
tests/fixtures/config/templated-project/git/common-config/zuul.d/projects.yaml
vendored
Normal file
|
@ -0,0 +1,14 @@
|
|||
- project:
|
||||
name: org/templated-project
|
||||
templates:
|
||||
- test-one-and-two
|
||||
|
||||
- project:
|
||||
name: org/layered-project
|
||||
templates:
|
||||
- test-one-and-two
|
||||
- test-three-and-four
|
||||
- test-five
|
||||
check:
|
||||
jobs:
|
||||
- project-test6
|
19
tests/fixtures/config/templated-project/git/common-config/zuul.d/templates.yaml
vendored
Normal file
19
tests/fixtures/config/templated-project/git/common-config/zuul.d/templates.yaml
vendored
Normal file
|
@ -0,0 +1,19 @@
|
|||
- project-template:
|
||||
name: test-one-and-two
|
||||
check:
|
||||
jobs:
|
||||
- project-test1
|
||||
- project-test2
|
||||
|
||||
- project-template:
|
||||
name: test-three-and-four
|
||||
check:
|
||||
jobs:
|
||||
- layered-project-test3
|
||||
- layered-project-test4
|
||||
|
||||
- project-template:
|
||||
name: test-five
|
||||
check:
|
||||
jobs:
|
||||
- layered-project-foo-test5
|
|
@ -1,94 +0,0 @@
|
|||
- pipeline:
|
||||
name: check
|
||||
manager: independent
|
||||
trigger:
|
||||
gerrit:
|
||||
- event: patchset-created
|
||||
success:
|
||||
gerrit:
|
||||
verified: 1
|
||||
failure:
|
||||
gerrit:
|
||||
verified: -1
|
||||
|
||||
- pipeline:
|
||||
name: gate
|
||||
manager: dependent
|
||||
success-message: Build succeeded (gate).
|
||||
trigger:
|
||||
gerrit:
|
||||
- event: comment-added
|
||||
approval:
|
||||
- approved: 1
|
||||
success:
|
||||
gerrit:
|
||||
verified: 2
|
||||
submit: true
|
||||
failure:
|
||||
gerrit:
|
||||
verified: -2
|
||||
start:
|
||||
gerrit:
|
||||
verified: 0
|
||||
precedence: high
|
||||
|
||||
- pipeline:
|
||||
name: post
|
||||
manager: independent
|
||||
trigger:
|
||||
gerrit:
|
||||
- event: ref-updated
|
||||
ref: ^(?!refs/).*$
|
||||
|
||||
- project-template:
|
||||
name: test-one-and-two
|
||||
check:
|
||||
jobs:
|
||||
- project-test1
|
||||
- project-test2
|
||||
|
||||
- project-template:
|
||||
name: test-three-and-four
|
||||
check:
|
||||
jobs:
|
||||
- layered-project-test3
|
||||
- layered-project-test4
|
||||
|
||||
- project-template:
|
||||
name: test-five
|
||||
check:
|
||||
jobs:
|
||||
- layered-project-foo-test5
|
||||
|
||||
- job:
|
||||
name: project-test1
|
||||
|
||||
- job:
|
||||
name: project-test2
|
||||
|
||||
- job:
|
||||
name: layered-project-test3
|
||||
|
||||
- job:
|
||||
name: layered-project-test4
|
||||
|
||||
- job:
|
||||
name: layered-project-foo-test5
|
||||
|
||||
- job:
|
||||
name: project-test6
|
||||
|
||||
- project:
|
||||
name: org/templated-project
|
||||
templates:
|
||||
- test-one-and-two
|
||||
|
||||
- project:
|
||||
name: org/layered-project
|
||||
templates:
|
||||
- test-one-and-two
|
||||
- test-three-and-four
|
||||
- test-five
|
||||
check:
|
||||
jobs:
|
||||
- project-test6
|
|
@ -12,6 +12,7 @@
|
|||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import textwrap
|
||||
|
||||
from tests.base import ZuulTestCase
|
||||
|
||||
|
@ -186,3 +187,40 @@ class TestTenantGroups3(TenantParserTestCase):
|
|||
project2_config.pipelines['check'].job_list.jobs)
|
||||
self.assertTrue('project2-job' in
|
||||
project2_config.pipelines['check'].job_list.jobs)
|
||||
|
||||
|
||||
class TestSplitConfig(ZuulTestCase):
|
||||
tenant_config_file = 'config/split-config/main.yaml'
|
||||
|
||||
def setup_config(self):
|
||||
super(TestSplitConfig, self).setup_config()
|
||||
|
||||
def test_split_config(self):
|
||||
tenant = self.sched.abide.tenants.get('tenant-one')
|
||||
self.assertIn('project-test1', tenant.layout.jobs)
|
||||
project_config = tenant.layout.project_configs.get(
|
||||
'review.example.com/org/project')
|
||||
self.assertIn('project-test1',
|
||||
project_config.pipelines['check'].job_list.jobs)
|
||||
project1_config = tenant.layout.project_configs.get(
|
||||
'review.example.com/org/project1')
|
||||
self.assertIn('project1-project2-integration',
|
||||
project1_config.pipelines['check'].job_list.jobs)
|
||||
|
||||
def test_dynamic_split_config(self):
|
||||
in_repo_conf = textwrap.dedent(
|
||||
"""
|
||||
- project:
|
||||
name: org/project1
|
||||
check:
|
||||
jobs:
|
||||
- project-test1
|
||||
""")
|
||||
file_dict = {'.zuul.d/gate.yaml': in_repo_conf}
|
||||
A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A',
|
||||
files=file_dict)
|
||||
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
|
||||
self.waitUntilSettled()
|
||||
# project1-project2-integration test removed, only want project-test1
|
||||
self.assertHistory([
|
||||
dict(name='project-test1', result='SUCCESS', changes='1,1')])
|
||||
|
|
|
@ -1106,7 +1106,8 @@ class TenantParser(object):
|
|||
job = merger.getFiles(
|
||||
project.source.connection.connection_name,
|
||||
project.name, 'master',
|
||||
files=['zuul.yaml', '.zuul.yaml'])
|
||||
files=['zuul.yaml', '.zuul.yaml'],
|
||||
dirs=['zuul.d', '.zuul.d'])
|
||||
job.source_context = model.SourceContext(project, 'master',
|
||||
'', True)
|
||||
jobs.append(job)
|
||||
|
@ -1134,7 +1135,8 @@ class TenantParser(object):
|
|||
job = merger.getFiles(
|
||||
project.source.connection.connection_name,
|
||||
project.name, branch,
|
||||
files=['zuul.yaml', '.zuul.yaml'])
|
||||
files=['zuul.yaml', '.zuul.yaml'],
|
||||
dirs=['zuul.d', '.zuul.d'])
|
||||
job.source_context = model.SourceContext(
|
||||
project, branch, '', False)
|
||||
jobs.append(job)
|
||||
|
@ -1147,15 +1149,19 @@ class TenantParser(object):
|
|||
TenantParser.log.debug("Waiting for cat job %s" % (job,))
|
||||
job.wait()
|
||||
loaded = False
|
||||
for fn in ['zuul.yaml', '.zuul.yaml']:
|
||||
if job.files.get(fn):
|
||||
# Don't load from more than one file in a repo-branch
|
||||
if loaded:
|
||||
files = sorted(job.files.keys())
|
||||
for conf_root in ['zuul.yaml', '.zuul.yaml', 'zuul.d', '.zuul.d']:
|
||||
for fn in files:
|
||||
fn_root = fn.split('/')[0]
|
||||
if fn_root != conf_root or not job.files.get(fn):
|
||||
continue
|
||||
# Don't load from more than configuration in a repo-branch
|
||||
if loaded and loaded != conf_root:
|
||||
TenantParser.log.warning(
|
||||
"Multiple configuration files in %s" %
|
||||
(job.source_context,))
|
||||
continue
|
||||
loaded = True
|
||||
loaded = conf_root
|
||||
job.source_context.path = fn
|
||||
TenantParser.log.info(
|
||||
"Loading configuration from %s" %
|
||||
|
@ -1328,28 +1334,50 @@ class ConfigLoader(object):
|
|||
branches = project.source.getProjectBranches(project)
|
||||
|
||||
for branch in branches:
|
||||
fns1 = []
|
||||
fns2 = []
|
||||
files_list = files.connections.get(
|
||||
project.source.connection.connection_name, {}).get(
|
||||
project.name, {}).get(branch, {}).keys()
|
||||
for fn in files_list:
|
||||
if fn.startswith("zuul.d/"):
|
||||
fns1.append(fn)
|
||||
if fn.startswith(".zuul.d/"):
|
||||
fns2.append(fn)
|
||||
|
||||
fns = ['zuul.yaml', '.zuul.yaml'] + sorted(fns1) + sorted(fns2)
|
||||
incdata = None
|
||||
for fn in ['zuul.yaml', '.zuul.yaml']:
|
||||
loaded = None
|
||||
for fn in fns:
|
||||
data = files.getFile(project.source.connection.connection_name,
|
||||
project.name, branch, fn)
|
||||
if data:
|
||||
break
|
||||
if data:
|
||||
source_context = model.SourceContext(project, branch,
|
||||
fn, trusted)
|
||||
if trusted:
|
||||
incdata = TenantParser._parseConfigProjectLayout(
|
||||
data, source_context)
|
||||
else:
|
||||
incdata = TenantParser._parseUntrustedProjectLayout(
|
||||
data, source_context)
|
||||
else:
|
||||
source_context = model.SourceContext(project, branch,
|
||||
fn, trusted)
|
||||
# Prevent mixing configuration source
|
||||
conf_root = fn.split('/')[0]
|
||||
if loaded and loaded != conf_root:
|
||||
TenantParser.log.warning(
|
||||
"Multiple configuration in %s" % source_context)
|
||||
continue
|
||||
loaded = conf_root
|
||||
|
||||
if trusted:
|
||||
incdata = TenantParser._parseConfigProjectLayout(
|
||||
data, source_context)
|
||||
else:
|
||||
incdata = TenantParser._parseUntrustedProjectLayout(
|
||||
data, source_context)
|
||||
|
||||
config.extend(incdata)
|
||||
|
||||
if not loaded:
|
||||
if trusted:
|
||||
incdata = project.unparsed_config
|
||||
else:
|
||||
incdata = project.unparsed_branch_config.get(branch)
|
||||
if incdata:
|
||||
config.extend(incdata)
|
||||
if incdata:
|
||||
config.extend(incdata)
|
||||
|
||||
def createDynamicLayout(self, tenant, files,
|
||||
include_config_projects=False):
|
||||
|
|
|
@ -641,7 +641,8 @@ class ExecutorServer(object):
|
|||
task.wait()
|
||||
with self.merger_lock:
|
||||
files = self.merger.getFiles(args['connection'], args['project'],
|
||||
args['branch'], args['files'])
|
||||
args['branch'], args['files'],
|
||||
args.get('dirs', []))
|
||||
result = dict(updated=True,
|
||||
files=files,
|
||||
zuul_url=self.zuul_url)
|
||||
|
@ -651,6 +652,7 @@ class ExecutorServer(object):
|
|||
args = json.loads(job.arguments)
|
||||
with self.merger_lock:
|
||||
ret = self.merger.mergeChanges(args['items'], args.get('files'),
|
||||
args.get('dirs', []),
|
||||
args.get('repo_state'))
|
||||
result = dict(merged=(ret is not None),
|
||||
zuul_url=self.zuul_url)
|
||||
|
|
|
@ -480,7 +480,7 @@ class PipelineManager(object):
|
|||
self.log.debug("Preparing dynamic layout for: %s" % item.change)
|
||||
return self._loadDynamicLayout(item)
|
||||
|
||||
def scheduleMerge(self, item, files=None):
|
||||
def scheduleMerge(self, item, files=None, dirs=None):
|
||||
build_set = item.current_build_set
|
||||
|
||||
if not hasattr(item.change, 'branch'):
|
||||
|
@ -490,12 +490,12 @@ class PipelineManager(object):
|
|||
build_set.merge_state = build_set.COMPLETE
|
||||
return True
|
||||
|
||||
self.log.debug("Scheduling merge for item %s (files: %s)" %
|
||||
(item, files))
|
||||
self.log.debug("Scheduling merge for item %s (files: %s, dirs: %s)" %
|
||||
(item, files, dirs))
|
||||
build_set = item.current_build_set
|
||||
build_set.merge_state = build_set.PENDING
|
||||
self.sched.merger.mergeChanges(build_set.merger_items,
|
||||
item.current_build_set, files,
|
||||
item.current_build_set, files, dirs,
|
||||
precedence=self.pipeline.precedence)
|
||||
return False
|
||||
|
||||
|
@ -506,7 +506,9 @@ class PipelineManager(object):
|
|||
if not build_set.ref:
|
||||
build_set.setConfiguration()
|
||||
if build_set.merge_state == build_set.NEW:
|
||||
return self.scheduleMerge(item, ['zuul.yaml', '.zuul.yaml'])
|
||||
return self.scheduleMerge(item,
|
||||
files=['zuul.yaml', '.zuul.yaml'],
|
||||
dirs=['zuul.d', '.zuul.d'])
|
||||
if build_set.merge_state == build_set.PENDING:
|
||||
return False
|
||||
if build_set.unable_to_merge:
|
||||
|
|
|
@ -108,19 +108,21 @@ class MergeClient(object):
|
|||
timeout=300)
|
||||
return job
|
||||
|
||||
def mergeChanges(self, items, build_set, files=None, repo_state=None,
|
||||
precedence=zuul.model.PRECEDENCE_NORMAL):
|
||||
def mergeChanges(self, items, build_set, files=None, dirs=None,
|
||||
repo_state=None, precedence=zuul.model.PRECEDENCE_NORMAL):
|
||||
data = dict(items=items,
|
||||
files=files,
|
||||
dirs=dirs,
|
||||
repo_state=repo_state)
|
||||
self.submitJob('merger:merge', data, build_set, precedence)
|
||||
|
||||
def getFiles(self, connection_name, project_name, branch, files,
|
||||
def getFiles(self, connection_name, project_name, branch, files, dirs=[],
|
||||
precedence=zuul.model.PRECEDENCE_HIGH):
|
||||
data = dict(connection=connection_name,
|
||||
project=project_name,
|
||||
branch=branch,
|
||||
files=files)
|
||||
files=files,
|
||||
dirs=dirs)
|
||||
job = self.submitJob('merger:cat', data, None, precedence)
|
||||
return job
|
||||
|
||||
|
|
|
@ -254,7 +254,7 @@ class Repo(object):
|
|||
origin.fetch()
|
||||
origin.fetch(tags=True)
|
||||
|
||||
def getFiles(self, files, branch=None, commit=None):
|
||||
def getFiles(self, files, dirs=[], branch=None, commit=None):
|
||||
ret = {}
|
||||
repo = self.createRepoObject()
|
||||
if branch:
|
||||
|
@ -266,6 +266,14 @@ class Repo(object):
|
|||
ret[fn] = tree[fn].data_stream.read().decode('utf8')
|
||||
else:
|
||||
ret[fn] = None
|
||||
if dirs:
|
||||
for dn in dirs:
|
||||
if dn not in tree:
|
||||
continue
|
||||
for blob in tree[dn].traverse():
|
||||
if blob.path.endswith(".yaml"):
|
||||
ret[blob.path] = blob.data_stream.read().decode(
|
||||
'utf-8')
|
||||
return ret
|
||||
|
||||
def deleteRemote(self, remote):
|
||||
|
@ -452,7 +460,7 @@ class Merger(object):
|
|||
return None
|
||||
return commit
|
||||
|
||||
def mergeChanges(self, items, files=None, repo_state=None):
|
||||
def mergeChanges(self, items, files=None, dirs=None, repo_state=None):
|
||||
# connection+project+branch -> commit
|
||||
recent = {}
|
||||
commit = None
|
||||
|
@ -470,9 +478,9 @@ class Merger(object):
|
|||
commit = self._mergeItem(item, recent, repo_state)
|
||||
if not commit:
|
||||
return None
|
||||
if files:
|
||||
if files or dirs:
|
||||
repo = self.getRepo(item['connection'], item['project'])
|
||||
repo_files = repo.getFiles(files, commit=commit)
|
||||
repo_files = repo.getFiles(files, dirs, commit=commit)
|
||||
read_files.append(dict(
|
||||
connection=item['connection'],
|
||||
project=item['project'],
|
||||
|
@ -483,6 +491,6 @@ class Merger(object):
|
|||
ret_recent[k] = v.hexsha
|
||||
return commit.hexsha, read_files, repo_state, ret_recent
|
||||
|
||||
def getFiles(self, connection_name, project_name, branch, files):
|
||||
def getFiles(self, connection_name, project_name, branch, files, dirs=[]):
|
||||
repo = self.getRepo(connection_name, project_name)
|
||||
return repo.getFiles(files, branch=branch)
|
||||
return repo.getFiles(files, dirs, branch=branch)
|
||||
|
|
|
@ -94,8 +94,9 @@ class MergeServer(object):
|
|||
|
||||
def merge(self, job):
|
||||
args = json.loads(job.arguments)
|
||||
ret = self.merger.mergeChanges(args['items'], args.get('files'),
|
||||
args.get('repo_state'))
|
||||
ret = self.merger.mergeChanges(
|
||||
args['items'], args.get('files'),
|
||||
args.get('dirs'), args.get('repo_state'))
|
||||
result = dict(merged=(ret is not None),
|
||||
zuul_url=self.zuul_url)
|
||||
if ret is None:
|
||||
|
@ -109,7 +110,8 @@ class MergeServer(object):
|
|||
args = json.loads(job.arguments)
|
||||
self.merger.updateRepo(args['connection'], args['project'])
|
||||
files = self.merger.getFiles(args['connection'], args['project'],
|
||||
args['branch'], args['files'])
|
||||
args['branch'], args['files'],
|
||||
args.get('dirs'))
|
||||
result = dict(updated=True,
|
||||
files=files,
|
||||
zuul_url=self.zuul_url)
|
||||
|
|
|
@ -1848,7 +1848,9 @@ class Ref(object):
|
|||
return set()
|
||||
|
||||
def updatesConfig(self):
|
||||
if 'zuul.yaml' in self.files or '.zuul.yaml' in self.files:
|
||||
if 'zuul.yaml' in self.files or '.zuul.yaml' in self.files or \
|
||||
[True for fn in self.files if fn.startswith("zuul.d/") or
|
||||
fn.startswith(".zuul.d/")]:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
|
Loading…
Reference in New Issue