zuul/zuul
Code Issues Proposed changes

Merge "Add image configuration object"

Browse Source
This commit is contained in:
Zuul
2024-06-26 23:52:18 +00:00
committed by Gerrit Code Review
parent 09908ae8ff 6cd360bc63
commit 563a7ec59c
5 changed files with 236 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

45
tests/base.py
View File

@@ -153,11 +153,12 @@ def iterate_timeout(max_seconds, purpose):
raise Exception("Timeout waiting for %s" % purpose)
def simple_layout(path, driver='gerrit'):
def simple_layout(path, driver='gerrit', enable_nodepool=False):
"""Specify a layout file for use by a test method.
:arg str path: The path to the layout file.
:arg str driver: The source driver to use, defaults to gerrit.
:arg bool enable_nodepool: Enable additional nodepool objects.
Some tests require only a very simple configuration. For those,
establishing a complete config directory hierachy is too much
@@ -170,10 +171,16 @@ def simple_layout(path, driver='gerrit'):
config-project called "common-config" and each "project" instance
referenced in the layout file will have a git repo automatically
initialized.
The enable_nodepool argument is a temporary facility for
convenience during the initial stages of the nodepool-in-zuul
work. It enables the additional nodepool config objects (which
are not otherwise enabled by default, but will be later).
"""
def decorator(test):
test.__simple_layout__ = (path, driver)
test.__enable_nodepool__ = enable_nodepool
return test
return decorator
@@ -2027,15 +2034,10 @@ class TestConfig:
def __init__(self, testobj):
test_name = testobj.id().split('.')[-1]
test = getattr(testobj, test_name)
self.simple_layout = None
self.gerrit_config = {}
self.never_capture = None
if hasattr(test, '__simple_layout__'):
self.simple_layout = getattr(test, '__simple_layout__')
if hasattr(test, '__gerrit_config__'):
self.gerrit_config = getattr(test, '__gerrit_config__')
if hasattr(test, '__never_capture__'):
self.gerrit_config = getattr(test, '__never_capture__')
self.simple_layout = getattr(test, '__simple_layout__', None)
self.gerrit_config = getattr(test, '__gerrit_config__', {})
self.never_capture = getattr(test, '__never_capture__', None)
self.enable_nodepool = getattr(test, '__enable_nodepool__', False)
self.changes = FakeChangeDB()
@@ -2445,13 +2447,32 @@ class ZuulTestCase(BaseTestCase):
data = f.read()
layout = yaml.safe_load(data)
files['zuul.yaml'] = data
config_projects = []
if self.test_config.enable_nodepool:
config_projects.append({
'org/common-config': {
'include': [
'image',
]}
})
else:
config_projects.append('org/common-config')
untrusted_projects = []
for item in layout:
if 'project' in item:
name = item['project']['name']
if name.startswith('^'):
continue
untrusted_projects.append(name)
if self.test_config.enable_nodepool:
untrusted_projects.append({
name: {
'include': [
'image',
]}
})
else:
untrusted_projects.append(name)
if self.init_repos:
self.init_repo(name)
self.addCommitToRepo(name, 'initial commit',
@@ -2476,7 +2497,7 @@ class ZuulTestCase(BaseTestCase):
'name': 'tenant-one',
'source': {
driver: {
'config-projects': ['org/common-config'],
'config-projects': config_projects,
'untrusted-projects': untrusted_projects}}}}]
f.write(yaml.dump(temp_config).encode('utf8'))
f.close()

89
tests/fixtures/layouts/nodepool.yaml vendored Normal file
View File

@@ -0,0 +1,89 @@
- pipeline:
name: check
manager: independent
trigger:
gerrit:
- event: patchset-created
success:
gerrit:
Verified: 1
failure:
gerrit:
Verified: -1
- pipeline:
name: gate
manager: dependent
success-message: Build succeeded (gate).
trigger:
gerrit:
- event: comment-added
approval:
- Approved: 1
success:
gerrit:
Verified: 2
submit: true
failure:
gerrit:
Verified: -2
start:
gerrit:
Verified: 0
precedence: high
- pipeline:
name: post
manager: independent
trigger:
gerrit:
- event: ref-updated
ref: ^(?!refs/).*$
- pipeline:
name: tag
manager: independent
trigger:
gerrit:
- event: ref-updated
ref: ^refs/tags/.*$
- job:
name: base
parent: null
run: playbooks/base.yaml
nodeset:
nodes:
- label: ubuntu-xenial
name: controller
- job:
name: check-job
run: playbooks/check.yaml
- job:
name: post-job
run: playbooks/post.yaml
- job:
name: tag-job
run: playbooks/tag.yaml
- project:
name: org/project
check:
jobs:
- check-job
gate:
jobs:
- check-job
post:
jobs:
- post-job
tag:
jobs:
- tag-job
- image:
name: debian
type: cloud

12
tests/unit/test_configloader.py
View File

@@ -1314,3 +1314,15 @@ class TestDefaultBranch(ZuulTestCase):
md = layout.getProjectMetadata(
'github.com/org/regex-override-project-develop')
self.assertEqual('develop', md.default_branch)
class TestNodepoolConfig(ZuulTestCase):
config_file = 'zuul-connections-gerrit-and-github.conf'
@simple_layout('layouts/nodepool.yaml', enable_nodepool=True)
def test_nodepool_config(self):
layout = self.scheds.first.sched.abide.tenants.get('tenant-one').layout
self.assertEqual(1, len(layout.images))
image = layout.images['debian']
self.assertEqual('debian', image.name)
self.assertEqual('cloud', image.type)

69
zuul/configloader.py
View File

@@ -260,7 +260,7 @@ class LocalAccumulator:
class ZuulSafeLoader(yaml.EncryptedLoader):
zuul_node_types = frozenset(('job', 'nodeset', 'secret', 'pipeline',
'project', 'project-template',
'semaphore', 'queue', 'pragma'))
'semaphore', 'queue', 'pragma', 'image'))
def __init__(self, stream, source_context):
wrapped_stream = io.StringIO(stream)
@@ -383,6 +383,30 @@ class PragmaParser(object):
for x in as_list(branches)]
class ImageParser(object):
image = {
'_source_context': model.SourceContext,
'_start_mark': model.ZuulMark,
vs.Required('name'): str,
vs.Required('type'): vs.Any('zuul', 'cloud'),
}
schema = vs.Schema(image)
def __init__(self, pcontext):
self.log = logging.getLogger("zuul.ImageParser")
self.pcontext = pcontext
def fromYaml(self, conf):
conf = copy_safe_config(conf)
self.schema(conf)
image = model.Image(conf['name'], conf['type'])
image.source_context = conf.get('_source_context')
image.start_mark = conf.get('_start_mark')
image.freeze()
return image
class NodeSetParser(object):
def __init__(self, pcontext):
self.log = logging.getLogger("zuul.NodeSetParser")
@@ -1569,6 +1593,7 @@ class ParseContext(object):
self.job_parser = JobParser(self)
self.semaphore_parser = SemaphoreParser(self)
self.queue_parser = QueueParser(self)
self.image_parser = ImageParser(self)
self.project_template_parser = ProjectTemplateParser(self)
self.project_parser = ProjectParser(self)
acc = LocalAccumulator(self.loading_errors)
@@ -1654,7 +1679,8 @@ class TenantParser(object):
self.unparsed_config_cache = unparsed_config_cache
classes = vs.Any('pipeline', 'job', 'semaphore', 'project',
'project-template', 'nodeset', 'secret', 'queue')
'project-template', 'nodeset', 'secret', 'queue',
'image')
project_dict = {str: {
'include': to_list(classes),
@@ -2031,6 +2057,8 @@ class TenantParser(object):
config_projects = []
untrusted_projects = []
# TODO: Add nodepool objects here (image, etc) when ready to
# use zuul-launcher.
default_include = frozenset(['pipeline', 'job', 'semaphore', 'project',
'secret', 'project-template', 'nodeset',
'queue'])
@@ -2405,6 +2433,15 @@ class TenantParser(object):
parsed_config.pipelines.append(
pcontext.pipeline_parser.fromYaml(config_pipeline))
for config_image in unparsed_config.images:
classes = self._getLoadClasses(tenant, config_image)
if 'image' not in classes:
continue
with pcontext.errorContext(stanza='image', conf=config_image):
with pcontext.accumulator.catchErrors():
parsed_config.images.append(
pcontext.image_parser.fromYaml(config_image))
for config_nodeset in unparsed_config.nodesets:
classes = self._getLoadClasses(tenant, config_nodeset)
if 'nodeset' not in classes:
@@ -2521,12 +2558,14 @@ class TenantParser(object):
for project_config in parsed_config.projects:
_cache('projects', project_config)
for image in parsed_config.images:
_cache('images', image)
def _addLayoutItems(self, layout, tenant, parsed_config,
parse_context, skip_pipelines=False,
skip_semaphores=False):
parse_context, dynamic_layout=False):
# TODO(jeblair): make sure everything needing
# reference_exceptions has it; add tests if needed.
if not skip_pipelines:
if not dynamic_layout:
for pipeline in parsed_config.pipelines:
with parse_context.errorContext(stanza='pipeline',
conf=pipeline):
@@ -2568,19 +2607,24 @@ class TenantParser(object):
with parse_context.accumulator.catchErrors():
pipeline.validateReferences(layout)
if skip_semaphores:
if dynamic_layout:
# We should not actually update the layout with new
# semaphores, but so that we can validate that the config
# is correct, create a shadow layout here to which we add
# new semaphores so validation is complete.
semaphore_layout = model.Layout(tenant)
shadow_layout = model.Layout(tenant)
else:
semaphore_layout = layout
shadow_layout = layout
for semaphore in parsed_config.semaphores:
with parse_context.errorContext(stanza='semaphore',
conf=semaphore):
with parse_context.accumulator.catchErrors():
semaphore_layout.addSemaphore(semaphore)
shadow_layout.addSemaphore(semaphore)
for image in parsed_config.images:
with parse_context.errorContext(stanza='image',
conf=image):
with parse_context.accumulator.catchErrors():
shadow_layout.addImage(image)
for queue in parsed_config.queues:
with parse_context.errorContext(stanza='queue', conf=queue):
@@ -3038,12 +3082,11 @@ class ConfigLoader(object):
# time. So we do not support dynamic semaphore
# configuration changes.
layout.semaphores = tenant.layout.semaphores
skip_pipelines = skip_semaphores = True
dynamic_layout = True
else:
skip_pipelines = skip_semaphores = False
dynamic_layout = False
self.tenant_parser._addLayoutItems(layout, tenant, config,
pcontext,
skip_pipelines=skip_pipelines,
skip_semaphores=skip_semaphores)
dynamic_layout=dynamic_layout)
return layout

47
zuul/model.py
View File

@@ -1447,6 +1447,40 @@ class ApiRoot(ConfigObject):
return f'<ApiRoot realm={self.default_auth_realm}>'
class Image(ConfigObject):
"""A zuul or cloud image.
Images are associated with labels and providers.
"""
def __init__(self, name, image_type):
super().__init__()
self.name = name
self.type = image_type
def __repr__(self):
return '<Image %s>' % (self.name,)
def __ne__(self, other):
return not self.__eq__(other)
def __eq__(self, other):
if not isinstance(other, Image):
return False
return (self.name == other.name and
self.type == other.type)
def toDict(self):
return {
'name': self.name,
'type': self.type
}
@classmethod
def fromDict(cls, data):
return cls(data["name"], data["type"])
class Node(ConfigObject):
"""A single node for use by a job.
@@ -7738,6 +7772,7 @@ class UnparsedConfig(object):
self.secrets = []
self.semaphores = []
self.queues = []
self.images = []
# The list of files/dirs which this represents.
self.files_examined = set()
@@ -7752,7 +7787,7 @@ class UnparsedConfig(object):
source_contexts = {}
for attr in ['pragmas', 'pipelines', 'jobs', 'project_templates',
'projects', 'nodesets', 'secrets', 'semaphores',
'queues']:
'queues', 'images']:
# Make a deep copy of each of our attributes
old_objlist = getattr(self, attr)
new_objlist = copy.deepcopy(old_objlist)
@@ -7789,6 +7824,7 @@ class UnparsedConfig(object):
self.secrets.extend(conf.secrets)
self.semaphores.extend(conf.semaphores)
self.queues.extend(conf.queues)
self.images.extend(conf.images)
return
if not isinstance(conf, list):
@@ -7820,6 +7856,8 @@ class UnparsedConfig(object):
self.queues.append(value)
elif key == 'pragma':
self.pragmas.append(value)
elif key == 'image':
self.images.append(value)
else:
raise ConfigItemUnknownError(item)
@@ -7838,6 +7876,7 @@ class ParsedConfig(object):
self.secrets = []
self.semaphores = []
self.queues = []
self.images = []
def copy(self):
r = ParsedConfig()
@@ -7851,6 +7890,7 @@ class ParsedConfig(object):
r.secrets = self.secrets[:]
r.semaphores = self.semaphores[:]
r.queues = self.queues[:]
r.images = self.images[:]
return r
def extend(self, conf):
@@ -7864,6 +7904,7 @@ class ParsedConfig(object):
self.secrets.extend(conf.secrets)
self.semaphores.extend(conf.semaphores)
self.queues.extend(conf.queues)
self.images.extend(conf.images)
for regex, projects in conf.projects_by_regex.items():
self.projects_by_regex.setdefault(regex, []).extend(projects)
return
@@ -7906,6 +7947,7 @@ class Layout(object):
self.secrets = {}
self.semaphores = {}
self.queues = {}
self.images = {}
self.loading_errors = LoadingErrors()
def getJob(self, name):
@@ -8015,6 +8057,9 @@ class Layout(object):
def addQueue(self, queue):
self._addIdenticalObject('Queue', self.queues, queue)
def addImage(self, image):
self._addIdenticalObject('Image', self.images, image)
def addPipeline(self, pipeline):
if pipeline.tenant is not self.tenant:
raise Exception("Pipeline created for tenant %s "