Add support for zuul.d configuration split

This change implements the zuul_split spec to support configuration split in
a zuul.d directory.

Change-Id: I6bc7250b2045b73dfba109aa0b2f1ba5d66752b2
changes/64/473764/10
Tristan Cacqueray 5 years ago
parent 2c414c1ca2
commit 829e617bac
  1. 2
      tests/fixtures/config/split-config/git/common-config/playbooks/project-test1.yaml
  2. 2
      tests/fixtures/config/split-config/git/common-config/zuul.d/jobs.yaml
  3. 5
      tests/fixtures/config/split-config/git/common-config/zuul.d/org-project.yaml
  4. 12
      tests/fixtures/config/split-config/git/common-config/zuul.d/pipelines.yaml
  5. 1
      tests/fixtures/config/split-config/git/org_project/README
  6. 7
      tests/fixtures/config/split-config/git/org_project1/.zuul.d/gate.yaml
  7. 2
      tests/fixtures/config/split-config/git/org_project1/.zuul.d/jobs.yaml
  8. 1
      tests/fixtures/config/split-config/git/org_project1/README
  9. 2
      tests/fixtures/config/split-config/git/org_project1/playbooks/project1-project2-integration.yaml
  10. 9
      tests/fixtures/config/split-config/main.yaml
  11. 17
      tests/fixtures/config/templated-project/git/common-config/zuul.d/jobs.yaml
  12. 41
      tests/fixtures/config/templated-project/git/common-config/zuul.d/pipelines.yaml
  13. 14
      tests/fixtures/config/templated-project/git/common-config/zuul.d/projects.yaml
  14. 19
      tests/fixtures/config/templated-project/git/common-config/zuul.d/templates.yaml
  15. 94
      tests/fixtures/config/templated-project/git/common-config/zuul.yaml
  16. 38
      tests/unit/test_configloader.py
  17. 70
      zuul/configloader.py
  18. 4
      zuul/executor/server.py
  19. 12
      zuul/manager/__init__.py
  20. 10
      zuul/merger/client.py
  21. 20
      zuul/merger/merger.py
  22. 8
      zuul/merger/server.py
  23. 4
      zuul/model.py

@ -0,0 +1,5 @@
- project:
name: org/project
check:
jobs:
- project-test1

@ -0,0 +1,12 @@
- pipeline:
name: check
manager: independent
trigger:
gerrit:
- event: patchset-created
success:
gerrit:
verified: 1
failure:
gerrit:
verified: -1

@ -0,0 +1,7 @@
- project:
name: org/project1
check:
jobs:
- project-test1
- project1-project2-integration:
dependencies: project-test1

@ -0,0 +1,2 @@
- job:
name: project1-project2-integration

@ -0,0 +1,9 @@
- tenant:
name: tenant-one
source:
gerrit:
config-projects:
- common-config
untrusted-projects:
- org/project
- org/project1

@ -0,0 +1,17 @@
- job:
name: project-test1
- job:
name: project-test2
- job:
name: layered-project-test3
- job:
name: layered-project-test4
- job:
name: layered-project-foo-test5
- job:
name: project-test6

@ -0,0 +1,41 @@
- pipeline:
name: check
manager: independent
trigger:
gerrit:
- event: patchset-created
success:
gerrit:
verified: 1
failure:
gerrit:
verified: -1
- pipeline:
name: gate
manager: dependent
success-message: Build succeeded (gate).
trigger:
gerrit:
- event: comment-added
approval:
- approved: 1
success:
gerrit:
verified: 2
submit: true
failure:
gerrit:
verified: -2
start:
gerrit:
verified: 0
precedence: high
- pipeline:
name: post
manager: independent
trigger:
gerrit:
- event: ref-updated
ref: ^(?!refs/).*$

@ -0,0 +1,14 @@
- project:
name: org/templated-project
templates:
- test-one-and-two
- project:
name: org/layered-project
templates:
- test-one-and-two
- test-three-and-four
- test-five
check:
jobs:
- project-test6

@ -0,0 +1,19 @@
- project-template:
name: test-one-and-two
check:
jobs:
- project-test1
- project-test2
- project-template:
name: test-three-and-four
check:
jobs:
- layered-project-test3
- layered-project-test4
- project-template:
name: test-five
check:
jobs:
- layered-project-foo-test5

@ -1,94 +0,0 @@
- pipeline:
name: check
manager: independent
trigger:
gerrit:
- event: patchset-created
success:
gerrit:
verified: 1
failure:
gerrit:
verified: -1
- pipeline:
name: gate
manager: dependent
success-message: Build succeeded (gate).
trigger:
gerrit:
- event: comment-added
approval:
- approved: 1
success:
gerrit:
verified: 2
submit: true
failure:
gerrit:
verified: -2
start:
gerrit:
verified: 0
precedence: high
- pipeline:
name: post
manager: independent
trigger:
gerrit:
- event: ref-updated
ref: ^(?!refs/).*$
- project-template:
name: test-one-and-two
check:
jobs:
- project-test1
- project-test2
- project-template:
name: test-three-and-four
check:
jobs:
- layered-project-test3
- layered-project-test4
- project-template:
name: test-five
check:
jobs:
- layered-project-foo-test5
- job:
name: project-test1
- job:
name: project-test2
- job:
name: layered-project-test3
- job:
name: layered-project-test4
- job:
name: layered-project-foo-test5
- job:
name: project-test6
- project:
name: org/templated-project
templates:
- test-one-and-two
- project:
name: org/layered-project
templates:
- test-one-and-two
- test-three-and-four
- test-five
check:
jobs:
- project-test6

@ -12,6 +12,7 @@
# License for the specific language governing permissions and limitations
# under the License.
import textwrap
from tests.base import ZuulTestCase
@ -186,3 +187,40 @@ class TestTenantGroups3(TenantParserTestCase):
project2_config.pipelines['check'].job_list.jobs)
self.assertTrue('project2-job' in
project2_config.pipelines['check'].job_list.jobs)
class TestSplitConfig(ZuulTestCase):
tenant_config_file = 'config/split-config/main.yaml'
def setup_config(self):
super(TestSplitConfig, self).setup_config()
def test_split_config(self):
tenant = self.sched.abide.tenants.get('tenant-one')
self.assertIn('project-test1', tenant.layout.jobs)
project_config = tenant.layout.project_configs.get(
'review.example.com/org/project')
self.assertIn('project-test1',
project_config.pipelines['check'].job_list.jobs)
project1_config = tenant.layout.project_configs.get(
'review.example.com/org/project1')
self.assertIn('project1-project2-integration',
project1_config.pipelines['check'].job_list.jobs)
def test_dynamic_split_config(self):
in_repo_conf = textwrap.dedent(
"""
- project:
name: org/project1
check:
jobs:
- project-test1
""")
file_dict = {'.zuul.d/gate.yaml': in_repo_conf}
A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A',
files=file_dict)
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
# project1-project2-integration test removed, only want project-test1
self.assertHistory([
dict(name='project-test1', result='SUCCESS', changes='1,1')])

@ -1106,7 +1106,8 @@ class TenantParser(object):
job = merger.getFiles(
project.source.connection.connection_name,
project.name, 'master',
files=['zuul.yaml', '.zuul.yaml'])
files=['zuul.yaml', '.zuul.yaml'],
dirs=['zuul.d', '.zuul.d'])
job.source_context = model.SourceContext(project, 'master',
'', True)
jobs.append(job)
@ -1134,7 +1135,8 @@ class TenantParser(object):
job = merger.getFiles(
project.source.connection.connection_name,
project.name, branch,
files=['zuul.yaml', '.zuul.yaml'])
files=['zuul.yaml', '.zuul.yaml'],
dirs=['zuul.d', '.zuul.d'])
job.source_context = model.SourceContext(
project, branch, '', False)
jobs.append(job)
@ -1147,15 +1149,19 @@ class TenantParser(object):
TenantParser.log.debug("Waiting for cat job %s" % (job,))
job.wait()
loaded = False
for fn in ['zuul.yaml', '.zuul.yaml']:
if job.files.get(fn):
# Don't load from more than one file in a repo-branch
if loaded:
files = sorted(job.files.keys())
for conf_root in ['zuul.yaml', '.zuul.yaml', 'zuul.d', '.zuul.d']:
for fn in files:
fn_root = fn.split('/')[0]
if fn_root != conf_root or not job.files.get(fn):
continue
# Don't load from more than configuration in a repo-branch
if loaded and loaded != conf_root:
TenantParser.log.warning(
"Multiple configuration files in %s" %
(job.source_context,))
continue
loaded = True
loaded = conf_root
job.source_context.path = fn
TenantParser.log.info(
"Loading configuration from %s" %
@ -1328,28 +1334,50 @@ class ConfigLoader(object):
branches = project.source.getProjectBranches(project)
for branch in branches:
fns1 = []
fns2 = []
files_list = files.connections.get(
project.source.connection.connection_name, {}).get(
project.name, {}).get(branch, {}).keys()
for fn in files_list:
if fn.startswith("zuul.d/"):
fns1.append(fn)
if fn.startswith(".zuul.d/"):
fns2.append(fn)
fns = ['zuul.yaml', '.zuul.yaml'] + sorted(fns1) + sorted(fns2)
incdata = None
for fn in ['zuul.yaml', '.zuul.yaml']:
loaded = None
for fn in fns:
data = files.getFile(project.source.connection.connection_name,
project.name, branch, fn)
if data:
break
if data:
source_context = model.SourceContext(project, branch,
fn, trusted)
if trusted:
incdata = TenantParser._parseConfigProjectLayout(
data, source_context)
else:
incdata = TenantParser._parseUntrustedProjectLayout(
data, source_context)
else:
source_context = model.SourceContext(project, branch,
fn, trusted)
# Prevent mixing configuration source
conf_root = fn.split('/')[0]
if loaded and loaded != conf_root:
TenantParser.log.warning(
"Multiple configuration in %s" % source_context)
continue
loaded = conf_root
if trusted:
incdata = TenantParser._parseConfigProjectLayout(
data, source_context)
else:
incdata = TenantParser._parseUntrustedProjectLayout(
data, source_context)
config.extend(incdata)
if not loaded:
if trusted:
incdata = project.unparsed_config
else:
incdata = project.unparsed_branch_config.get(branch)
if incdata:
config.extend(incdata)
if incdata:
config.extend(incdata)
def createDynamicLayout(self, tenant, files,
include_config_projects=False):

@ -641,7 +641,8 @@ class ExecutorServer(object):
task.wait()
with self.merger_lock:
files = self.merger.getFiles(args['connection'], args['project'],
args['branch'], args['files'])
args['branch'], args['files'],
args.get('dirs', []))
result = dict(updated=True,
files=files,
zuul_url=self.zuul_url)
@ -651,6 +652,7 @@ class ExecutorServer(object):
args = json.loads(job.arguments)
with self.merger_lock:
ret = self.merger.mergeChanges(args['items'], args.get('files'),
args.get('dirs', []),
args.get('repo_state'))
result = dict(merged=(ret is not None),
zuul_url=self.zuul_url)

@ -480,7 +480,7 @@ class PipelineManager(object):
self.log.debug("Preparing dynamic layout for: %s" % item.change)
return self._loadDynamicLayout(item)
def scheduleMerge(self, item, files=None):
def scheduleMerge(self, item, files=None, dirs=None):
build_set = item.current_build_set
if not hasattr(item.change, 'branch'):
@ -490,12 +490,12 @@ class PipelineManager(object):
build_set.merge_state = build_set.COMPLETE
return True
self.log.debug("Scheduling merge for item %s (files: %s)" %
(item, files))
self.log.debug("Scheduling merge for item %s (files: %s, dirs: %s)" %
(item, files, dirs))
build_set = item.current_build_set
build_set.merge_state = build_set.PENDING
self.sched.merger.mergeChanges(build_set.merger_items,
item.current_build_set, files,
item.current_build_set, files, dirs,
precedence=self.pipeline.precedence)
return False
@ -506,7 +506,9 @@ class PipelineManager(object):
if not build_set.ref:
build_set.setConfiguration()
if build_set.merge_state == build_set.NEW:
return self.scheduleMerge(item, ['zuul.yaml', '.zuul.yaml'])
return self.scheduleMerge(item,
files=['zuul.yaml', '.zuul.yaml'],
dirs=['zuul.d', '.zuul.d'])
if build_set.merge_state == build_set.PENDING:
return False
if build_set.unable_to_merge:

@ -108,19 +108,21 @@ class MergeClient(object):
timeout=300)
return job
def mergeChanges(self, items, build_set, files=None, repo_state=None,
precedence=zuul.model.PRECEDENCE_NORMAL):
def mergeChanges(self, items, build_set, files=None, dirs=None,
repo_state=None, precedence=zuul.model.PRECEDENCE_NORMAL):
data = dict(items=items,
files=files,
dirs=dirs,
repo_state=repo_state)
self.submitJob('merger:merge', data, build_set, precedence)
def getFiles(self, connection_name, project_name, branch, files,
def getFiles(self, connection_name, project_name, branch, files, dirs=[],
precedence=zuul.model.PRECEDENCE_HIGH):
data = dict(connection=connection_name,
project=project_name,
branch=branch,
files=files)
files=files,
dirs=dirs)
job = self.submitJob('merger:cat', data, None, precedence)
return job

@ -254,7 +254,7 @@ class Repo(object):
origin.fetch()
origin.fetch(tags=True)
def getFiles(self, files, branch=None, commit=None):
def getFiles(self, files, dirs=[], branch=None, commit=None):
ret = {}
repo = self.createRepoObject()
if branch:
@ -266,6 +266,14 @@ class Repo(object):
ret[fn] = tree[fn].data_stream.read().decode('utf8')
else:
ret[fn] = None
if dirs:
for dn in dirs:
if dn not in tree:
continue
for blob in tree[dn].traverse():
if blob.path.endswith(".yaml"):
ret[blob.path] = blob.data_stream.read().decode(
'utf-8')
return ret
def deleteRemote(self, remote):
@ -452,7 +460,7 @@ class Merger(object):
return None
return commit
def mergeChanges(self, items, files=None, repo_state=None):
def mergeChanges(self, items, files=None, dirs=None, repo_state=None):
# connection+project+branch -> commit
recent = {}
commit = None
@ -470,9 +478,9 @@ class Merger(object):
commit = self._mergeItem(item, recent, repo_state)
if not commit:
return None
if files:
if files or dirs:
repo = self.getRepo(item['connection'], item['project'])
repo_files = repo.getFiles(files, commit=commit)
repo_files = repo.getFiles(files, dirs, commit=commit)
read_files.append(dict(
connection=item['connection'],
project=item['project'],
@ -483,6 +491,6 @@ class Merger(object):
ret_recent[k] = v.hexsha
return commit.hexsha, read_files, repo_state, ret_recent
def getFiles(self, connection_name, project_name, branch, files):
def getFiles(self, connection_name, project_name, branch, files, dirs=[]):
repo = self.getRepo(connection_name, project_name)
return repo.getFiles(files, branch=branch)
return repo.getFiles(files, dirs, branch=branch)

@ -94,8 +94,9 @@ class MergeServer(object):
def merge(self, job):
args = json.loads(job.arguments)
ret = self.merger.mergeChanges(args['items'], args.get('files'),
args.get('repo_state'))
ret = self.merger.mergeChanges(
args['items'], args.get('files'),
args.get('dirs'), args.get('repo_state'))
result = dict(merged=(ret is not None),
zuul_url=self.zuul_url)
if ret is None:
@ -109,7 +110,8 @@ class MergeServer(object):
args = json.loads(job.arguments)
self.merger.updateRepo(args['connection'], args['project'])
files = self.merger.getFiles(args['connection'], args['project'],
args['branch'], args['files'])
args['branch'], args['files'],
args.get('dirs'))
result = dict(updated=True,
files=files,
zuul_url=self.zuul_url)

@ -1848,7 +1848,9 @@ class Ref(object):
return set()
def updatesConfig(self):
if 'zuul.yaml' in self.files or '.zuul.yaml' in self.files:
if 'zuul.yaml' in self.files or '.zuul.yaml' in self.files or \
[True for fn in self.files if fn.startswith("zuul.d/") or
fn.startswith(".zuul.d/")]:
return True
return False

Loading…
Cancel
Save