zuul/zuul
Code Issues Proposed changes

Merge "Add parameterized builds (server)"

Browse Source
This commit is contained in:
Zuul authored and Gerrit Code Review committed 2026-06-04 18:34:16 +00:00
2 parents b0e51f27c8 9a748ac777
commit a504586684
11 files changed
+450 -9
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

tests/fixtures/layouts/parameterized-builds-not-allowed.yaml
Vendored
+40
View File
@@ -0,0 +1,40 @@
- pipeline:
name: on-demand
manager: independent
# allow-parameters: false
success:
gerrit:
Verified: 1
failure:
gerrit:
Verified: -1
- job:
name: base
parent: null
run: playbooks/base.yaml
nodeset:
nodes:
- label: ubuntu-xenial
name: controller
- job:
name: deploy-app
run: playbooks/deploy.yaml
- project:
name: org/project
on-demand:
parameters:
- name: target_cluster
description: Which cluster to deploy to
type: selection
values:
- production
- staging
- dev
- name: version
description: What version to deploy
type: string
jobs:
- deploy-app
tests/fixtures/layouts/parameterized-builds.yaml
Vendored
+40
View File
@@ -0,0 +1,40 @@
- pipeline:
name: on-demand
manager: independent
allow-parameters: true
success:
gerrit:
Verified: 1
failure:
gerrit:
Verified: -1
- job:
name: base
parent: null
run: playbooks/base.yaml
nodeset:
nodes:
- label: ubuntu-xenial
name: controller
- job:
name: deploy-app
run: playbooks/deploy.yaml
- project:
name: org/project
on-demand:
parameters:
- name: target_cluster
description: Which cluster to deploy to
type: selection
values:
- production
- staging
- dev
- name: version
description: What version to deploy
type: string
jobs:
- deploy-app
tests/unit/test_configloader.py
+10
View File
@@ -1970,3 +1970,13 @@ class TestNodepoolConfig(LauncherBaseTestCase):
self.assertEqual(1, len(layout.loading_errors))
self.assertIn('more than the max-retain-count',
layout.loading_errors[0].error)
class TestParameterizedBuilds(ZuulTestCase):
@simple_layout('layouts/parameterized-builds-not-allowed.yaml')
def test_parameterized_builds_not_allowed(self):
layout = self.scheds.first.sched.abide.tenants.get('tenant-one').layout
self.assertEqual(1, len(layout.loading_errors))
self.assertIn('does not allow parameters',
layout.loading_errors[0].error)
tests/unit/test_connection.py
+1 -1
View File
@@ -92,7 +92,7 @@ class TestSQLConnectionMysql(ZuulTestCase):
buildset_event_table = table_prefix + 'zuul_buildset_event'
self.assertEqual(9, len(insp.get_columns(ref_table)))
self.assertEqual(12, len(insp.get_columns(buildset_table)))
self.assertEqual(14, len(insp.get_columns(buildset_table)))
self.assertEqual(2, len(insp.get_columns(buildset_ref_table)))
self.assertEqual(15, len(insp.get_columns(build_table)))
self.assertEqual(5, len(insp.get_columns(artifact_table)))
tests/unit/test_web.py
+78
View File
@@ -955,9 +955,11 @@ class TestWeb(BaseTestWeb):
'pipelines': [{
'name': 'check',
'jobs': jobs,
'parameters': [],
}, {
'name': 'gate',
'jobs': jobs,
'parameters': [],
}, {'name': 'post',
'jobs': [[
{'abstract': False,
@@ -1010,6 +1012,7 @@ class TestWeb(BaseTestWeb):
'workspace_checkout': True,
}
]],
'parameters': [],
}
]
}]
@@ -4197,6 +4200,81 @@ class TestTenantScopedWebApi(BaseTestWeb):
dict(name='project-test2', result='SUCCESS', changes='2,1'),
], ordered=False)
@simple_layout('layouts/parameterized-builds.yaml')
def test_web_parameterized_build(self):
token = self._getToken(['tenant-one'])
resp = self.post_url(
"api/tenant/tenant-one/project/org/project/freeze-pipeline",
json={
'ref': 'refs/heads/master',
'pipeline': 'on-demand',
})
data = resp.json()
self.assertEqual([
{'default': None,
'description': 'Which cluster to deploy to',
'name': 'target_cluster',
'required': False,
'type': 'selection',
'values': ['production', 'staging', 'dev']},
{'default': None,
'description': 'What version to deploy',
'name': 'version',
'required': False,
'type': 'string',
'values': None},
], data['parameters'])
expected_params = {
'target_cluster': 'production',
'version': '1.0',
}
resp = self.post_url(
"api/tenant/tenant-one/project/org/project/enqueue",
headers={'Authorization': 'Bearer %s' % token},
json={
'ref': 'refs/heads/master',
'pipeline': 'on-demand',
'parameters': expected_params,
})
self.assertEqual(200, resp.status_code)
self.waitUntilSettled()
params = self.builds[0].job.variables
self.assertEqual(expected_params, params)
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
self.waitUntilSettled()
self.assertHistory([
dict(name='deploy-app', result='SUCCESS', ref='refs/heads/master'),
], ordered=False)
buildsets = self.get_url("api/tenant/tenant-one/buildsets").json()
self.assertEqual(1, len(buildsets))
buildset = self.get_url(
"api/tenant/tenant-one/buildset/%s" % buildsets[0]['uuid']).json()
self.assertEqual(expected_params, buildset['parameters'])
self.assertEqual('testuser', buildset['user_id'])
@simple_layout('layouts/parameterized-builds.yaml')
def test_web_parameterized_build_change(self):
token = self._getToken(['tenant-one'])
expected_params = {
'target_cluster': 'production',
'version': '1.0',
}
resp = self.post_url(
"api/tenant/tenant-one/project/org/project/enqueue",
headers={'Authorization': 'Bearer %s' % token},
json={
'ref': 'refs/changes/01/1/1',
'pipeline': 'on-demand',
'parameters': expected_params,
})
self.assertEqual(400, resp.status_code)
self.assertIn('supported only with branches or tags', resp.text)
class TestTenantScopedWebApiWithAccessRules(TestTenantScopedWebApi):
config_file = 'zuul-admin-web.conf'
zuul/configloader.py
+39 -1
View File
@@ -1327,10 +1327,23 @@ class ProjectTemplateParser(object):
job = {str: vs.Any(str, JobParser.job_attributes)}
job_list = [vs.Any(str, job)]
parameter_types = vs.Any(str, int, float, bool)
parameter = {
vs.Required('name'): ansible_var_name,
vs.Required('type'): vs.Any(
'string', 'selection', 'multiple-selection', 'bool'),
'description': str,
'values': [parameter_types],
'default': to_list(parameter_types),
'required': bool,
}
pipeline_contents = {
'debug': bool,
'fail-fast': bool,
'jobs': job_list,
'parameters': to_list(parameter),
}
project = {
@@ -1366,6 +1379,15 @@ class ProjectTemplateParser(object):
self.parseJobList(
conf_pipeline.get('jobs', []),
source_context, start_mark, project_pipeline.job_list)
for conf_param in conf_pipeline.get('parameters', []):
param = model.ProjectParameterConfig()
param.name = conf_param['name']
param.type = conf_param['type']
param.description = conf_param.get('description')
param.values = conf_param.get('values')
param.default = conf_param.get('default')
param.required = conf_param.get('required', False)
project_pipeline.parameters[param.name] = param
# If this project definition is in a place where it
# should get implied branch matchers, set it.
@@ -1412,10 +1434,23 @@ class ProjectParser(object):
job = {str: vs.Any(str, JobParser.job_attributes)}
job_list = [vs.Any(str, job)]
parameter_types = vs.Any(str, int, float, bool)
parameter = {
vs.Required('name'): ansible_var_name,
vs.Required('type'): vs.Any(
'string', 'selection', 'multiple-selection', 'bool'),
'description': str,
'values': [parameter_types],
'default': to_list(parameter_types),
'required': bool,
}
pipeline_contents = {
'debug': bool,
'fail-fast': bool,
'jobs': job_list
'jobs': job_list,
'parameters': to_list(parameter),
}
project = {
@@ -1554,6 +1589,7 @@ class PipelineParser(object):
pipeline = {vs.Required('name'): str,
vs.Required('manager'): manager,
'allow-other-connections': bool,
'allow-parameters': bool,
'precedence': precedence,
'supercedes': to_list(str),
'description': str,
@@ -1596,6 +1632,8 @@ class PipelineParser(object):
pipeline.start_mark = conf['_start_mark']
pipeline.allow_other_connections = conf.get(
'allow-other-connections', True)
pipeline.allow_parameters = conf.get(
'allow-parameters', False)
pipeline.description = conf.get('description')
pipeline.supercedes = as_list(conf.get('supercedes', []))
zuul/driver/sql/alembic/versions/1385a2593eb2_buildset_parameters.py
+41
View File
@@ -0,0 +1,41 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""buildset_parameters
Revision ID: 1385a2593eb2
Revises: 268b2da7a66e
Create Date: 2026-05-28 10:03:39.013680
"""
# revision identifiers, used by Alembic.
revision = '1385a2593eb2'
down_revision = '268b2da7a66e'
branch_labels = None
depends_on = None
from alembic import op
import sqlalchemy as sa
def upgrade(table_prefix=''):
op.add_column(
table_prefix + 'zuul_buildset',
sa.Column('parameters', sa.JSON(), nullable=True))
op.add_column(
table_prefix + 'zuul_buildset',
sa.Column('user_id', sa.String(255), nullable=True))
def downgrade():
raise Exception("Downgrades not supported")
zuul/driver/sql/sqlconnection.py
+2
View File
@@ -817,6 +817,8 @@ class SQLConnection(BaseConnection):
last_build_end_time = sa.Column(sa.DateTime, nullable=True)
updated = sa.Column(sa.DateTime, nullable=True)
queue_item_uuid = sa.Column(sa.String(36), nullable=True)
parameters = sa.Column(sa.JSON(), nullable=True)
user_id = sa.Column(sa.String(255), nullable=True)
refs = orm.relationship(
RefModel,
zuul/driver/sql/sqlreporter.py
+4
View File
@@ -56,11 +56,13 @@ class SQLReporter(BaseReporter):
def _createBuildset(self, db, buildset):
event_id = None
event_timestamp = None
event_user_id = None
item = buildset.item
if item.event is not None:
event_id = getattr(item.event, "zuul_event_id", None)
event_timestamp = datetime.datetime.fromtimestamp(
item.event.timestamp, tz=datetime.timezone.utc)
event_user_id = getattr(item.event, "user_id", None)
db_buildset = db.createBuildSet(
uuid=buildset.uuid,
@@ -70,6 +72,8 @@ class SQLReporter(BaseReporter):
event_timestamp=event_timestamp,
queue_item_uuid=item.uuid,
updated=datetime.datetime.utcnow(),
user_id=event_user_id,
parameters=item.parameters,
)
for change in item.changes:
ref = db.getOrCreateRef(
zuul/model.py
+88 -3
View File
@@ -569,6 +569,7 @@ class Pipeline(object):
def __init__(self, name):
self.name = name
self.allow_other_connections = True
self.allow_parameters = False
self.connections = []
self.source_context = None
self.start_mark = None
@@ -1275,14 +1276,17 @@ class ChangeQueue(zkobject.ZKObject):
"Unable to identify triggering ref from event %s",
event)
event_info = EventInfo.fromEvent(event, event_ref_cache_key)
parameters = getattr(event, 'parameters', None)
else:
event_info = None
parameters = None
item = QueueItem.new(self.zk_context,
queue=self,
changes=changes,
event=event_info,
span_info=span_info,
enqueue_time=enqueue_time,
parameters=parameters,
debug=debug)
self.enqueueItem(item)
return item
@@ -7134,6 +7138,7 @@ class EventInfo:
self.image_names = None
self.image_upload_uuid = None
self.image_build_uuid = None
self.user_id = None
@classmethod
def fromEvent(cls, event, event_ref_key):
@@ -7148,6 +7153,7 @@ class EventInfo:
tinfo.image_names = getattr(event, 'image_names', None)
tinfo.image_upload_uuid = getattr(event, 'image_upload_uuid', None)
tinfo.image_build_uuid = getattr(event, 'image_build_uuid', None)
tinfo.user_id = getattr(event, 'user_id', None)
return tinfo
@classmethod
@@ -7160,6 +7166,7 @@ class EventInfo:
tinfo.image_names = d.get("image_names")
tinfo.image_upload_uuid = d.get("image_upload_uuid")
tinfo.image_build_uuid = d.get("image_build_uuid")
tinfo.user_id = d.get("user_id")
return tinfo
def isSuperset(self, other):
@@ -7195,6 +7202,7 @@ class EventInfo:
"image_names": self.image_names,
"image_upload_uuid": self.image_upload_uuid,
"image_build_uuid": self.image_build_uuid,
"user_id": self.user_id,
}
@@ -7238,6 +7246,8 @@ class QueueItem(zkobject.ZKObject):
_cached_sql_results={},
event=None, # Info about the event that lead to this queue item
debug=False,
user_id=None,
parameters=None,
reporter_jobs_state=None,
# Change.cache_key -> sha for any changes in this item
# that were merged
@@ -7321,6 +7331,7 @@ class QueueItem(zkobject.ZKObject):
"type": event_type,
"data": self.event.toDict(),
},
"parameters": self.parameters,
"dynamic_state": self.dynamic_state,
"first_job_start_time": self.first_job_start_time,
"debug": self.debug,
@@ -9097,11 +9108,14 @@ class ChangeManagementEvent(ManagementEvent):
:arg str newrev: optional, the new revision
:arg dict orig_ref: optional, the cache key reference of the
ref of the original triggering event
:arg str user_id: optional, event originator user_id
:arg dict parameters: optional, pipeline parameters
"""
def __init__(self, tenant_name, pipeline_name, project_hostname,
project_name, change=None, ref=None, oldrev=None,
newrev=None, orig_ref=None):
newrev=None, orig_ref=None, user_id=None,
parameters=None):
super().__init__()
self.type = None
self.tenant_name = tenant_name
@@ -9120,6 +9134,8 @@ class ChangeManagementEvent(ManagementEvent):
self.timestamp = time.time()
span = trace.get_current_span()
self.span_context = tracing.getSpanContext(span)
self.user_id = user_id
self.parameters = parameters
@property
def canonical_project_name(self):
@@ -9139,6 +9155,8 @@ class ChangeManagementEvent(ManagementEvent):
d["timestamp"] = self.timestamp
d["span_context"] = self.span_context
d["orig_ref"] = self.orig_ref
d["user_id"] = self.user_id
d["parameters"] = self.parameters
return d
def updateFromDict(self, d):
@@ -9159,6 +9177,8 @@ class ChangeManagementEvent(ManagementEvent):
data.get("oldrev"),
data.get("newrev"),
data.get("orig_ref"),
data.get("user_id"),
data.get("parameters"),
)
event.updateFromDict(data)
return event
@@ -9696,6 +9716,28 @@ class TenantProjectConfig(object):
return True
class ProjectParameterConfig(ConfigObject):
# Represents a project parameter
def __init__(self):
super().__init__()
self.name = None
self.description = None
self.type = None
self.values = None
self.default = None
self.required = False
def toDict(self):
return {
'name': self.name,
'description': self.description,
'type': self.type,
'values': self.values,
'default': self.default,
'required': self.required,
}
class ProjectPipelineConfig(ConfigObject):
# Represents a project cofiguration in the context of a pipeline
def __init__(self):
@@ -9705,6 +9747,7 @@ class ProjectPipelineConfig(ConfigObject):
self.debug_messages = []
self.fail_fast = None
self.variables = {}
self.parameters = {}
def addDebug(self, msg):
self.debug_messages.append(msg)
@@ -9717,6 +9760,9 @@ class ProjectPipelineConfig(ConfigObject):
if self.fail_fast is None:
self.fail_fast = other.fail_fast
self.job_list.inheritFrom(other.job_list)
for p in other.parameters.values():
if p.name not in self.parameters:
self.parameters[p.name] = p
def updateVariables(self, other):
# We need to keep this separate to update() because we wish to
@@ -9725,8 +9771,9 @@ class ProjectPipelineConfig(ConfigObject):
self.variables = Job._deepUpdate(self.variables, other)
def toDict(self):
d = {}
return d
return {
'parameters': [x.toDict() for x in self.parameters.values()],
}
class ProjectConfig(ConfigObject):
@@ -10678,10 +10725,21 @@ class Layout(object):
self.pipeline_managers[manager.pipeline.name] = manager
def _checkAddPPC(self, pipeline_name, ppc):
manager = self.pipeline_managers.get(pipeline_name)
# TODO: Consaider raising errors on incorrect pipeline names
if manager:
pipeline = manager.pipeline
if ppc.parameters and not pipeline.allow_parameters:
raise Exception("Pipeline does not allow parameters")
def addProjectTemplate(self, project_template):
for job in project_template.embeddedJobs():
self._checkAddJob(job)
for pipeline_name, ppc in project_template.pipelines.items():
self._checkAddPPC(pipeline_name, ppc)
template_list = self.project_templates.get(project_template.name)
if template_list is not None:
reference = template_list[0]
@@ -10710,6 +10768,9 @@ class Layout(object):
for job in project_config.embeddedJobs():
self._checkAddJob(job)
for pipeline_name, ppc in project_config.pipelines.items():
self._checkAddPPC(pipeline_name, ppc)
if project_canonical_name in self.project_configs:
self.project_configs[project_canonical_name].append(project_config)
else:
@@ -10911,6 +10972,25 @@ class Layout(object):
raise NoMatchingParentError()
return jobs
def collectParameters(self, item, ppc):
# Filter supplied parameters against the parameter
# specifications in the ppc
parameters = {}
if not item.parameters:
return parameters
for (k, v) in item.parameters.items():
param_def = ppc.parameters.get(k)
if param_def:
if param_def.type == 'multiple-selection':
if isinstance(v, list) and set(v) <= set(param_def.values):
parameters[k] = v
elif param_def.type == 'selection':
if v in param_def.values:
parameters[k] = v
else:
parameters[k] = v
return parameters
def extendJobGraph(self, context, item, change, ppc, job_graph,
skip_file_matcher, redact_secrets_and_keys,
debug_messages, pending_errors):
@@ -10958,6 +11038,11 @@ class Layout(object):
final_job.name = variant.name
final_job.name = jobname
# Don't accept any parameters that aren't specified in our PPC
parameters = self.collectParameters(item, ppc)
# Pipeline parameters behave like project variables, but
# have higher precedence
final_job.updateProjectVariables(parameters)
# Now merge variables set from this parent ppc
# (i.e. project+templates) directly into the job vars
final_job.updateProjectVariables(ppc.variables)
zuul/web/__init__.py
+107 -4
View File
@@ -65,7 +65,9 @@ from zuul.model import (
PromoteEvent,
ProviderNode,
QueueItem,
Ref,
SystemAttributes,
Tag,
UnparsedAbideConfig,
WebInfo,
)
@@ -357,6 +359,8 @@ class BuildsetConverter:
'queue_item_uuid': buildset.queue_item_uuid,
'first_build_start_time': start,
'last_build_end_time': end,
'user_id': buildset.user_id,
'parameters': buildset.parameters,
'refs': [
RefConverter.toDict(ref)
for ref in buildset.refs
@@ -379,6 +383,8 @@ class BuildsetConverter:
'event_timestamp': str,
'first_build_start_time': str,
'last_build_end_time': str,
'user_id': str,
'parameters': dict,
'refs': [
RefConverter.schema()
],
@@ -807,6 +813,42 @@ class SystemEventConverter:
return Prop('The system event', ret)
class ProjectPipelineConverter:
# A class to encapsulate the conversion of project pipeline
# config objects to API output.
def toDict(ppc):
parameters = []
if ppc:
for p in ppc.parameters.values():
parameters.append({
'name': p.name,
'type': p.type,
'description': p.description,
'values': p.values,
'default': p.default,
'required': p.required,
})
ret = {
'parameters': parameters,
}
return ret
def schema():
parameter = {
'name': str,
'type': str,
'description': str,
'values': [str],
'default': str,
'required': bool,
}
ret = {
'parameters': [parameter],
}
return Prop('The project pipeline config', ret)
class APIError(cherrypy.HTTPError):
def __init__(self, code, json_doc=None, headers=None):
self._headers = headers or {}
@@ -1523,13 +1565,21 @@ class ZuulWebAPI(object):
if manager is None:
raise cherrypy.HTTPError(400, 'Unknown pipeline')
if body.get('parameters'):
ref = body.get('ref', '')
if not (ref.startswith('refs/heads') or
ref.startswith('refs/tags')):
raise cherrypy.HTTPError(
400, 'Parameters are supported only with branches or tags')
if 'change' in body:
return self._enqueue(tenant, project, manager.pipeline,
body['change'])
elif all(p in body for p in ['ref', 'oldrev', 'newrev']):
elif 'ref' in body:
return self._enqueue_ref(tenant, project,
manager.pipeline, body['ref'],
body['oldrev'], body['newrev'])
body.get('oldrev'), body.get('newrev'),
auth.uid, body.get('parameters'))
else:
raise cherrypy.HTTPError(400, 'Invalid request body')
@@ -1543,11 +1593,13 @@ class ZuulWebAPI(object):
return True
def _enqueue_ref(self, tenant, project, pipeline, ref, oldrev, newrev):
def _enqueue_ref(self, tenant, project, pipeline, ref, oldrev,
newrev, user_id, parameters):
event = EnqueueEvent(tenant.name, pipeline.name,
project.canonical_hostname, project.name,
change=None, ref=ref, oldrev=oldrev,
newrev=newrev)
newrev=newrev, user_id=user_id,
parameters=parameters)
event.zuul_event_id = get_zuul_request_id()
self.zuulweb.pipeline_management_events[tenant.name][
pipeline.name].put(event)
@@ -3376,6 +3428,50 @@ class ZuulWebAPI(object):
return item, change
@cherrypy.expose
@cherrypy.tools.save_params()
@cherrypy.tools.json_in()
@cherrypy.tools.json_out(content_type='application/json; charset=utf-8')
@cherrypy.tools.handle_options()
@cherrypy.tools.check_tenant_auth()
@openapi_response(
code=200,
content_type='application/json',
description='Returns the project pipeline config',
schema=ProjectPipelineConverter.schema(),
)
def project_freeze_pipeline(self, tenant_name, tenant, auth,
project_name):
body = cherrypy.request.json
if 'ref' not in body:
raise cherrypy.HTTPError(400, 'ref is required')
if 'pipeline' not in body:
raise cherrypy.HTTPError(400, 'pipeline is required')
project = self._getProjectOrRaise(tenant, project_name)
manager = tenant.layout.pipeline_managers.get(body['pipeline'])
if not manager:
raise cherrypy.HTTPError(404, 'Unknown pipeline')
# refs/heads/...
ref = body['ref'].split('/', 2)
if ref[1] == 'heads':
change = Branch(project)
change.branch = ref[2]
elif ref[1] == 'tags':
change = Tag(project)
change.tag = ref[2]
else:
change = Ref(project)
change.ref = body['ref']
change.cache_stat = FakeCacheKey()
with LocalZKContext(self.log) as context:
queue = ChangeQueue.new(context, manager=manager)
item = QueueItem.new(context, queue=queue, changes=[change])
ppc = tenant.layout.getProjectPipelineConfig(item, change)
return ProjectPipelineConverter.toDict(ppc)
class StaticHandler(object):
def __init__(self, root):
@@ -3593,6 +3689,13 @@ class ZuulWeb(object):
'api',
'/api/tenant/{tenant_name}/project/{project_name:.*}/dequeue',
controller=api, action='dequeue')
route_map.connect(
'api',
'/api/tenant/{tenant_name}/project/{project_name:.*}'
'/freeze-pipeline',
controller=api,
action='project_freeze_pipeline',
)
route_map.connect('api',
'/api/tenant/{tenant_name}/autohold/{request_id}',
controller=api,