Browse Source

Merge "Get executor job params"

changes/56/783556/1
Zuul 3 months ago
committed by Gerrit Code Review
parent
commit
3095910660
6 changed files with 378 additions and 166 deletions
  1. +100
    -0
      tests/unit/test_web.py
  2. +12
    -164
      zuul/executor/client.py
  3. +196
    -0
      zuul/executor/common.py
  4. +5
    -2
      zuul/model.py
  5. +35
    -0
      zuul/rpclistener.py
  6. +30
    -0
      zuul/web/__init__.py

+ 100
- 0
tests/unit/test_web.py View File

@ -989,6 +989,96 @@ class TestWeb(BaseTestWeb):
}
self.assertIn(expected, resp.json())
def test_freeze_job(self):
resp = self.get_url(
"api/tenant/tenant-one/pipeline/check"
"/project/org/project1/branch/master/freeze-job/"
"project-test1")
job_params = {
'job': 'project-test1',
'ansible_version': '2.9',
'timeout': None,
'post_timeout': None,
'items': [],
'projects': [],
'branch': 'master',
'cleanup_playbooks': [],
'groups': [],
'nodes': [{
'comment': None,
'hold_job': None,
'label': 'label1',
'name': ['controller'],
'state': 'unknown'
}],
'override_branch': None,
'override_checkout': None,
'repo_state': {},
'playbooks': [{
'connection': 'gerrit',
'project': 'common-config',
'branch': 'master',
'trusted': True,
'roles': [{
'target_name': 'common-config',
'type': 'zuul',
'project_canonical_name':
'review.example.com/common-config',
'implicit': True,
'project_default_branch': 'master',
'connection': 'gerrit',
'project': 'common-config',
}],
'secrets': {},
'path': 'playbooks/project-test1.yaml',
}],
'pre_playbooks': [],
'post_playbooks': [],
'ssh_keys': [],
'vars': {},
'extra_vars': {},
'host_vars': {},
'group_vars': {},
'zuul': {
'_inheritance_path': [
'<Job base branches: None source: '
'common-config/zuul.yaml@master#53>',
'<Job project-test1 branches: None source: '
'common-config/zuul.yaml@master#66>',
'<Job project-test1 branches: None source: '
'common-config/zuul.yaml@master#138>',
'<Job project-test1 branches: None source: '
'common-config/zuul.yaml@master#53>'],
'build': '00000000000000000000000000000000',
'buildset': None,
'branch': 'master',
'ref': None,
'pipeline': 'check',
'post_review': False,
'job': 'project-test1',
'voting': True,
'project': {
'name': 'org/project1',
'short_name': 'project1',
'canonical_hostname': 'review.example.com',
'canonical_name': 'review.example.com/org/project1',
'src_dir': 'src/review.example.com/org/project1',
},
'tenant': 'tenant-one',
'timeout': None,
'jobtags': [],
'branch': 'master',
'projects': {},
'items': [],
'child_jobs': [],
'event_id': None,
},
}
self.assertEqual(job_params, resp.json())
class TestWebMultiTenant(BaseTestWeb):
tenant_config_file = 'config/multi-tenant/main.yaml'
@ -1015,6 +1105,16 @@ class TestWebSecrets(BaseTestWeb):
secret = {'name': 'project1_secret', 'alias': 'secret_name'}
self.assertEqual([secret], run[0]['secrets'])
def test_freeze_job_redacted(self):
# Test that ssh_keys and secrets are redacted
resp = self.get_url(
"api/tenant/tenant-one/pipeline/check"
"/project/org/project1/branch/master/freeze-job/"
"project1-secret").json()
self.assertEqual(
{'secret_name': 'REDACTED'}, resp['playbooks'][0]['secrets'])
self.assertEqual('REDACTED', resp['ssh_keys'][0]['key'])
class TestInfo(ZuulDBTestCase, BaseTestWeb):


+ 12
- 164
zuul/executor/client.py View File

@ -15,12 +15,12 @@
import gear
import json
import logging
import os
import time
import threading
from uuid import uuid4
import zuul.model
import zuul.executor.common
from zuul.lib.config import get_default
from zuul.lib.gear_utils import getGearmanFunctions
from zuul.lib.jsonutil import json_dumps
@ -140,7 +140,6 @@ class ExecutorClient(object):
def execute(self, job, item, pipeline, dependent_changes=[],
merger_items=[]):
log = get_annotated_logger(self.log, item.event)
tenant = pipeline.tenant
uuid = str(uuid4().hex)
nodeset = item.current_build_set.getJobNodeSet(job.name)
log.info(
@ -148,165 +147,13 @@ class ExecutorClient(object):
"with dependent changes %s",
job, uuid, nodeset, item.change, dependent_changes)
project = dict(
name=item.change.project.name,
short_name=item.change.project.name.split('/')[-1],
canonical_hostname=item.change.project.canonical_hostname,
canonical_name=item.change.project.canonical_name,
src_dir=os.path.join('src', item.change.project.canonical_name),
)
zuul_params = dict(build=uuid,
buildset=item.current_build_set.uuid,
ref=item.change.ref,
pipeline=pipeline.name,
post_review=pipeline.post_review,
job=job.name,
voting=job.voting,
project=project,
tenant=tenant.name,
timeout=job.timeout,
event_id=item.event.zuul_event_id,
jobtags=sorted(job.tags),
_inheritance_path=list(job.inheritance_path))
if job.artifact_data:
zuul_params['artifacts'] = job.artifact_data
if job.override_checkout:
zuul_params['override_checkout'] = job.override_checkout
if hasattr(item.change, 'branch'):
zuul_params['branch'] = item.change.branch
if hasattr(item.change, 'tag'):
zuul_params['tag'] = item.change.tag
if hasattr(item.change, 'number'):
zuul_params['change'] = str(item.change.number)
if hasattr(item.change, 'url'):
zuul_params['change_url'] = item.change.url
if hasattr(item.change, 'patchset'):
zuul_params['patchset'] = str(item.change.patchset)
if hasattr(item.change, 'message'):
zuul_params['message'] = item.change.message
if (hasattr(item.change, 'oldrev') and item.change.oldrev
and item.change.oldrev != '0' * 40):
zuul_params['oldrev'] = item.change.oldrev
if (hasattr(item.change, 'newrev') and item.change.newrev
and item.change.newrev != '0' * 40):
zuul_params['newrev'] = item.change.newrev
zuul_params['projects'] = {} # Set below
zuul_params['items'] = dependent_changes
zuul_params['child_jobs'] = list(item.job_graph.getDirectDependentJobs(
job.name))
params = dict()
params['job'] = job.name
params['timeout'] = job.timeout
params['post_timeout'] = job.post_timeout
params['items'] = merger_items
params['projects'] = []
if hasattr(item.change, 'branch'):
params['branch'] = item.change.branch
else:
params['branch'] = None
params['override_branch'] = job.override_branch
params['override_checkout'] = job.override_checkout
params['repo_state'] = item.current_build_set.repo_state
params['ansible_version'] = job.ansible_version
def make_playbook(playbook):
d = playbook.toDict()
for role in d['roles']:
if role['type'] != 'zuul':
continue
project_metadata = item.layout.getProjectMetadata(
role['project_canonical_name'])
if project_metadata:
role['project_default_branch'] = \
project_metadata.default_branch
else:
role['project_default_branch'] = 'master'
role_trusted, role_project = item.layout.tenant.getProject(
role['project_canonical_name'])
role_connection = role_project.source.connection
role['connection'] = role_connection.connection_name
role['project'] = role_project.name
return d
if job.name != 'noop':
params['playbooks'] = [make_playbook(x) for x in job.run]
params['pre_playbooks'] = [make_playbook(x) for x in job.pre_run]
params['post_playbooks'] = [make_playbook(x) for x in job.post_run]
params['cleanup_playbooks'] = [make_playbook(x)
for x in job.cleanup_run]
nodes = []
for node in nodeset.getNodes():
n = node.toDict()
n.update(dict(name=node.name, label=node.label))
nodes.append(n)
params['nodes'] = nodes
params['groups'] = [group.toDict() for group in nodeset.getGroups()]
params['ssh_keys'] = []
if pipeline.post_review:
params['ssh_keys'].append(dict(
name='%s project key' % item.change.project.canonical_name,
key=item.change.project.private_ssh_key))
params['vars'] = job.combined_variables
params['extra_vars'] = job.extra_variables
params['host_vars'] = job.host_variables
params['group_vars'] = job.group_variables
params['zuul'] = zuul_params
projects = set()
required_projects = set()
def make_project_dict(project, override_branch=None,
override_checkout=None):
project_metadata = item.layout.getProjectMetadata(
project.canonical_name)
if project_metadata:
project_default_branch = project_metadata.default_branch
else:
project_default_branch = 'master'
connection = project.source.connection
return dict(connection=connection.connection_name,
name=project.name,
canonical_name=project.canonical_name,
override_branch=override_branch,
override_checkout=override_checkout,
default_branch=project_default_branch)
if job.required_projects:
for job_project in job.required_projects.values():
(trusted, project) = tenant.getProject(
job_project.project_name)
if project is None:
raise Exception("Unknown project %s" %
(job_project.project_name,))
params['projects'].append(
make_project_dict(project,
job_project.override_branch,
job_project.override_checkout))
projects.add(project)
required_projects.add(project)
for change in dependent_changes:
# We have to find the project this way because it may not
# be registered in the tenant (ie, a foreign project).
source = self.sched.connections.getSourceByCanonicalHostname(
change['project']['canonical_hostname'])
project = source.getProject(change['project']['name'])
if project not in projects:
params['projects'].append(make_project_dict(project))
projects.add(project)
for p in projects:
zuul_params['projects'][p.canonical_name] = (dict(
name=p.name,
short_name=p.name.split('/')[-1],
# Duplicate this into the dict too, so that iterating
# project.values() is easier for callers
canonical_name=p.canonical_name,
canonical_hostname=p.canonical_hostname,
src_dir=os.path.join('src', p.canonical_name),
required=(p in required_projects),
))
params['zuul_event_id'] = item.event.zuul_event_id
params = zuul.executor.common.construct_gearman_params(
uuid, self.sched, nodeset,
job, item, pipeline, dependent_changes, merger_items,
redact_secrets_and_keys=False)
# TODO: deprecate and remove this variable?
params["zuul"]["_inheritance_path"] = list(job.inheritance_path)
build = Build(job, uuid, zuul_event_id=item.event.zuul_event_id)
build.parameters = params
build.nodeset = nodeset
@ -323,7 +170,7 @@ class ExecutorClient(object):
# Update zuul attempts after addBuild above to ensure build_set
# is up to date.
attempts = build.build_set.getTries(job.name)
zuul_params['attempts'] = attempts
params["zuul"]['attempts'] = attempts
functions = getGearmanFunctions(self.gearman)
function_name = 'executor:execute'
@ -331,8 +178,9 @@ class ExecutorClient(object):
# availability zone we can get executor_zone from only the first
# node.
executor_zone = None
if nodes and nodes[0].get('attributes'):
executor_zone = nodes[0]['attributes'].get('executor-zone')
if params["nodes"] and params["nodes"][0].get('attributes'):
executor_zone = params[
"nodes"][0]['attributes'].get('executor-zone')
if executor_zone:
_fname = '%s:%s' % (


+ 196
- 0
zuul/executor/common.py View File

@ -0,0 +1,196 @@
# Copyright 2018 SUSE Linux GmbH
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import os
def construct_gearman_params(uuid, sched, nodeset, job, item, pipeline,
dependent_changes=[], merger_items=[],
redact_secrets_and_keys=True):
"""Returns a list of all the parameters needed to build a job.
These parameters may be passed to zuul-executors (via gearman) to perform
the job itself.
Alternatively they contain enough information to load into another build
environment - for example, a local runner.
"""
tenant = pipeline.tenant
project = dict(
name=item.change.project.name,
short_name=item.change.project.name.split('/')[-1],
canonical_hostname=item.change.project.canonical_hostname,
canonical_name=item.change.project.canonical_name,
src_dir=os.path.join('src', item.change.project.canonical_name),
)
zuul_params = dict(
build=uuid,
buildset=item.current_build_set.uuid,
ref=item.change.ref,
pipeline=pipeline.name,
post_review=pipeline.post_review,
job=job.name,
voting=job.voting,
project=project,
tenant=tenant.name,
timeout=job.timeout,
event_id=item.event.zuul_event_id if item.event else None,
jobtags=sorted(job.tags),
_inheritance_path=list(job.inheritance_path))
if job.artifact_data:
zuul_params['artifacts'] = job.artifact_data
if job.override_checkout:
zuul_params['override_checkout'] = job.override_checkout
if hasattr(item.change, 'branch'):
zuul_params['branch'] = item.change.branch
if hasattr(item.change, 'tag'):
zuul_params['tag'] = item.change.tag
if hasattr(item.change, 'number'):
zuul_params['change'] = str(item.change.number)
if hasattr(item.change, 'url'):
zuul_params['change_url'] = item.change.url
if hasattr(item.change, 'patchset'):
zuul_params['patchset'] = str(item.change.patchset)
if hasattr(item.change, 'message'):
zuul_params['message'] = item.change.message
if (hasattr(item.change, 'oldrev') and item.change.oldrev
and item.change.oldrev != '0' * 40):
zuul_params['oldrev'] = item.change.oldrev
if (hasattr(item.change, 'newrev') and item.change.newrev
and item.change.newrev != '0' * 40):
zuul_params['newrev'] = item.change.newrev
zuul_params['projects'] = {} # Set below
zuul_params['items'] = dependent_changes
zuul_params['child_jobs'] = list(item.job_graph.getDirectDependentJobs(
job.name))
params = dict()
params['job'] = job.name
params['timeout'] = job.timeout
params['post_timeout'] = job.post_timeout
params['items'] = merger_items
params['projects'] = []
if hasattr(item.change, 'branch'):
params['branch'] = item.change.branch
else:
params['branch'] = None
params['override_branch'] = job.override_branch
params['override_checkout'] = job.override_checkout
params['repo_state'] = item.current_build_set.repo_state
params['ansible_version'] = job.ansible_version
def make_playbook(playbook):
d = playbook.toDict(redact_secrets=redact_secrets_and_keys)
for role in d['roles']:
if role['type'] != 'zuul':
continue
project_metadata = item.layout.getProjectMetadata(
role['project_canonical_name'])
if project_metadata:
role['project_default_branch'] = \
project_metadata.default_branch
else:
role['project_default_branch'] = 'master'
role_trusted, role_project = item.layout.tenant.getProject(
role['project_canonical_name'])
role_connection = role_project.source.connection
role['connection'] = role_connection.connection_name
role['project'] = role_project.name
return d
if job.name != 'noop':
params['playbooks'] = [make_playbook(x) for x in job.run]
params['pre_playbooks'] = [make_playbook(x) for x in job.pre_run]
params['post_playbooks'] = [make_playbook(x) for x in job.post_run]
params['cleanup_playbooks'] = [make_playbook(x)
for x in job.cleanup_run]
nodes = []
for node in nodeset.getNodes():
n = node.toDict()
n.update(dict(name=node.name, label=node.label))
nodes.append(n)
params['nodes'] = nodes
params['groups'] = [group.toDict() for group in nodeset.getGroups()]
params['ssh_keys'] = []
if pipeline.post_review:
if redact_secrets_and_keys:
ssh_key = "REDACTED"
else:
ssh_key = item.change.project.private_ssh_key
params['ssh_keys'].append(dict(
name='%s project key' % item.change.project.canonical_name,
key=ssh_key))
params['vars'] = job.combined_variables
params['extra_vars'] = job.extra_variables
params['host_vars'] = job.host_variables
params['group_vars'] = job.group_variables
params['zuul'] = zuul_params
projects = set()
required_projects = set()
def make_project_dict(project, override_branch=None,
override_checkout=None):
project_metadata = item.layout.getProjectMetadata(
project.canonical_name)
if project_metadata:
project_default_branch = project_metadata.default_branch
else:
project_default_branch = 'master'
connection = project.source.connection
return dict(connection=connection.connection_name,
name=project.name,
canonical_name=project.canonical_name,
override_branch=override_branch,
override_checkout=override_checkout,
default_branch=project_default_branch)
if job.required_projects:
for job_project in job.required_projects.values():
(trusted, project) = tenant.getProject(
job_project.project_name)
if project is None:
raise Exception("Unknown project %s" %
(job_project.project_name,))
params['projects'].append(
make_project_dict(project,
job_project.override_branch,
job_project.override_checkout))
projects.add(project)
required_projects.add(project)
for change in dependent_changes:
# We have to find the project this way because it may not
# be registered in the tenant (ie, a foreign project).
source = sched.connections.getSourceByCanonicalHostname(
change['project']['canonical_hostname'])
project = source.getProject(change['project']['name'])
if project not in projects:
params['projects'].append(make_project_dict(project))
projects.add(project)
for p in projects:
zuul_params['projects'][p.canonical_name] = (dict(
name=p.name,
short_name=p.name.split('/')[-1],
# Duplicate this into the dict too, so that iterating
# project.values() is easier for callers
canonical_name=p.canonical_name,
canonical_hostname=p.canonical_hostname,
src_dir=os.path.join('src', p.canonical_name),
required=(p in required_projects),
))
if item.event:
params['zuul_event_id'] = item.event.zuul_event_id
return params

+ 5
- 2
zuul/model.py View File

@ -1075,11 +1075,14 @@ class PlaybookContext(ConfigObject):
if s.name not in current_names]
self.decrypted_secrets = self.decrypted_secrets + tuple(new_secrets)
def toDict(self):
def toDict(self, redact_secrets=True):
# Render to a dict to use in passing json to the executor
secrets = {}
for secret in self.decrypted_secrets:
secrets[secret.name] = secret.secret_data
if redact_secrets:
secrets[secret.name] = 'REDACTED'
else:
secrets[secret.name] = secret.secret_data
return dict(
connection=self.source_context.project.connection_name,
project=self.source_context.project.name,


+ 35
- 0
zuul/rpclistener.py View File

@ -19,6 +19,8 @@ import time
from abc import ABCMeta
from typing import List
import zuul.executor.common
from zuul import model
from zuul.connection import BaseConnection
from zuul.lib import encryption
@ -156,6 +158,7 @@ class RPCListener(RPCListenerBase):
'project_get',
'project_list',
'project_freeze_jobs',
'project_freeze_job',
'pipeline_list',
'key_get',
'config_errors_list',
@ -481,6 +484,38 @@ class RPCListener(RPCListenerBase):
gear_job.sendWorkComplete(json.dumps(output))
def handle_project_freeze_job(self, gear_job):
args = json.loads(gear_job.arguments)
tenant = self.sched.abide.tenants.get(args.get("tenant"))
project = None
pipeline = None
if tenant:
(trusted, project) = tenant.getProject(args.get("project"))
pipeline = tenant.layout.pipelines.get(args.get("pipeline"))
if not project or not pipeline:
gear_job.sendWorkComplete(json.dumps(None))
return
change = model.Branch(project)
change.branch = args.get("branch", "master")
queue = model.ChangeQueue(pipeline)
item = model.QueueItem(queue, change, None)
item.layout = tenant.layout
item.freezeJobGraph(skip_file_matcher=True)
job = item.job_graph.jobs.get(args.get("job"))
if not job:
gear_job.sendWorkComplete(json.dumps(None))
return
# TODO: check if this is frozen?
nodeset = job.nodeset
job.setBase(tenant.layout)
uuid = '0' * 32
params = zuul.executor.common.construct_gearman_params(
uuid, self.sched, nodeset,
job, item, pipeline)
gear_job.sendWorkComplete(json.dumps(params, cls=ZuulJSONEncoder))
def handle_allowed_labels_get(self, job):
args = json.loads(job.arguments)
tenant = self.sched.abide.tenants.get(args.get("tenant"))


+ 30
- 0
zuul/web/__init__.py View File

@ -1122,6 +1122,30 @@ class ZuulWebAPI(object):
resp.headers['Access-Control-Allow-Origin'] = '*'
return ret
@cherrypy.expose
@cherrypy.tools.save_params()
@cherrypy.tools.json_out(content_type='application/json; charset=utf-8')
def project_freeze_job(self, tenant, pipeline, project, branch, job):
# TODO(jhesketh): Allow a canonical change/item to be passed in which
# would return the job with any in-change modifications.
job = self.rpc.submitJob(
'zuul:project_freeze_job',
{
'tenant': tenant,
'project': project,
'pipeline': pipeline,
'branch': branch,
'job': job
}
)
ret = json.loads(job.data[0])
if not ret:
raise cherrypy.HTTPError(404)
resp = cherrypy.response
resp.headers['Access-Control-Allow-Origin'] = '*'
return ret
class StaticHandler(object):
def __init__(self, root):
@ -1325,6 +1349,12 @@ class ZuulWeb(object):
'/project/{project:.*}/branch/{branch:.*}/freeze-jobs',
controller=api, action='project_freeze_jobs'
)
route_map.connect(
'api',
'/api/tenant/{tenant}/pipeline/{pipeline}'
'/project/{project:.*}/branch/{branch:.*}/freeze-job/{job}',
controller=api, action='project_freeze_job'
)
route_map.connect('api', '/api/tenant/{tenant}/pipelines',
controller=api, action='pipelines')
route_map.connect('api', '/api/tenant/{tenant}/labels',


Loading…
Cancel
Save