zuul
/
zuul
Code Issues Proposed changes

Merge branch 'master' into feature/zuulv3 

Conflicts:
	zuul/model.py
	zuul/scheduler.py

Change-Id: I2973bfae65b3658549dc13aa3ea0efe60669ba8e
This commit is contained in:
Joshua Hesketh 2016-03-11 13:14:28 +11:00
parent c38a278a52 87a801fb1d
commit dc7820cf88
14 changed files with 152 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
doc/source/zuul.rst
View File

@ -765,6 +765,12 @@ each job as it builds a list from the project specification.
Boolean value (``true`` or ``false``) that indicates whatever
a job is voting or not. Default: ``true``.
**tags (optional)**
A list of arbitrary strings which will be associated with the job.
Can be used by the parameter-function to alter behavior based on
their presence on a job. If the job name is a regular expression,
tags will accumulate on jobs that match.
**parameter-function (optional)**
Specifies a function that should be applied to the parameters before
the job is launched. The function should be defined in a python file

1
requirements.txt
View File

@ -1,6 +1,5 @@
pbr>=1.1.0
argparse
PyYAML>=3.1.0
Paste
WebOb>=1.2.3

1
tests/base.py
View File

@ -623,6 +623,7 @@ class FakeBuild(threading.Thread):
BuildHistory(name=self.name, number=self.number,
result=result, changes=changes, node=self.node,
uuid=self.unique, description=self.description,
parameters=self.parameters,
pipeline=self.parameters['ZUUL_PIPELINE'])
)

42
tests/fixtures/layout-tags.yaml vendored Normal file
View File

@ -0,0 +1,42 @@
includes:
- python-file: tags_custom_functions.py
pipelines:
- name: check
manager: IndependentPipelineManager
trigger:
gerrit:
- event: patchset-created
success:
gerrit:
verified: 1
failure:
gerrit:
verified: -1
jobs:
- name: ^.*$
parameter-function: apply_tags
- name: ^.*-merge$
failure-message: Unable to merge change
hold-following-changes: true
tags: merge
- name: project1-merge
tags:
- project1
- extratag
projects:
- name: org/project1
check:
- project1-merge:
- project1-test1
- project1-test2
- project1-project2-integration
- name: org/project2
check:
- project2-merge:
- project2-test1
- project2-test2
- project1-project2-integration

5
tests/fixtures/layout.yaml vendored
View File

@ -123,6 +123,7 @@ jobs:
- name: ^.*-merge$
failure-message: Unable to merge change
hold-following-changes: true
tags: merge
- name: nonvoting-project-test2
voting: false
- name: project-testfile
@ -136,6 +137,10 @@ jobs:
mutex: test-mutex
- name: mutex-two
mutex: test-mutex
- name: project1-merge
tags:
- project1
- extratag
project-templates:
- name: test-one-and-two

2
tests/fixtures/tags_custom_functions.py vendored Normal file
View File

@ -0,0 +1,2 @@
def apply_tags(item, job, params):
params['BUILD_TAGS'] = ' '.join(sorted(job.tags))

75
tests/test_scheduler.py
View File

@ -699,8 +699,8 @@ class TestScheduler(ZuulTestCase):
# triggering events. Since it will have the changes cached
# already (without approvals), we need to clear the cache
# first.
source = self.sched.layout.pipelines['gate'].source
source.maintainCache([])
for connection in self.connections.values():
connection.maintainCache([])
self.worker.hold_jobs_in_build = True
A.addApproval('APRV', 1)
@ -797,7 +797,6 @@ class TestScheduler(ZuulTestCase):
A.addApproval('APRV', 1)
a = source._getChange(1, 2, refresh=True)
self.assertTrue(source.canMerge(a, mgr.getSubmitAllowNeeds()))
source.maintainCache([])
def test_build_configuration(self):
"Test that zuul merges the right commits for testing"
@ -2611,6 +2610,53 @@ class TestScheduler(ZuulTestCase):
# Ensure the removed job was not included in the report.
self.assertNotIn('project1-project2-integration', A.messages[0])
def test_double_live_reconfiguration_shared_queue(self):
# This was a real-world regression. A change is added to
# gate; a reconfigure happens, a second change which depends
# on the first is added, and a second reconfiguration happens.
# Ensure that both changes merge.
# A failure may indicate incorrect caching or cleaning up of
# references during a reconfiguration.
self.worker.hold_jobs_in_build = True
A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
B = self.fake_gerrit.addFakeChange('org/project1', 'master', 'B')
B.setDependsOn(A, 1)
A.addApproval('CRVW', 2)
B.addApproval('CRVW', 2)
# Add the parent change.
self.fake_gerrit.addEvent(A.addApproval('APRV', 1))
self.waitUntilSettled()
self.worker.release('.*-merge')
self.waitUntilSettled()
# Reconfigure (with only one change in the pipeline).
self.sched.reconfigure(self.config)
self.waitUntilSettled()
# Add the child change.
self.fake_gerrit.addEvent(B.addApproval('APRV', 1))
self.waitUntilSettled()
self.worker.release('.*-merge')
self.waitUntilSettled()
# Reconfigure (with both in the pipeline).
self.sched.reconfigure(self.config)
self.waitUntilSettled()
self.worker.hold_jobs_in_build = False
self.worker.release()
self.waitUntilSettled()
self.assertEqual(len(self.history), 8)
self.assertEqual(A.data['status'], 'MERGED')
self.assertEqual(A.reported, 2)
self.assertEqual(B.data['status'], 'MERGED')
self.assertEqual(B.reported, 2)
def test_live_reconfiguration_del_project(self):
# Test project deletion from layout
# while changes are enqueued
@ -2747,6 +2793,25 @@ class TestScheduler(ZuulTestCase):
self.assertEqual(B.data['status'], 'MERGED')
self.assertEqual(B.reported, 2)
def test_tags(self):
"Test job tags"
self.config.set('zuul', 'layout_config',
'tests/fixtures/layout-tags.yaml')
self.sched.reconfigure(self.config)
A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
results = {'project1-merge': 'extratag merge project1',
'project2-merge': 'merge'}
for build in self.history:
self.assertEqual(results.get(build.name, ''),
build.parameters.get('BUILD_TAGS'))
def test_timer(self):
"Test that a periodic job is triggered"
self.worker.hold_jobs_in_build = True
@ -3656,8 +3721,8 @@ For CI problems and help debugging, contact ci@example.org"""
self.assertEqual(A.data['status'], 'NEW')
self.assertEqual(B.data['status'], 'NEW')
source = self.sched.layout.pipelines['gate'].source
source.maintainCache([])
for connection in self.connections.values():
connection.maintainCache([])
self.worker.hold_jobs_in_build = True
B.addApproval('APRV', 1)

8
zuul/configloader.py
View File

@ -61,6 +61,7 @@ class JobParser(object):
'success-url': str,
'voting': bool,
'mutex': str,
'tags': to_list(str),
'branches': to_list(str),
'files': to_list(str),
'swift': to_list(swift),
@ -83,6 +84,13 @@ class JobParser(object):
job.post_run = as_list(conf.get('post-run', job.post_run))
job.voting = conf.get('voting', True)
job.mutex = conf.get('mutex', None)
tags = conf.get('tags')
if tags:
# Tags are merged via a union rather than a
# destructive copy because they are intended to
# accumulate onto any previously applied tags from
# metajobs.
job.tags = job.tags.union(set(tags))
job.failure_message = conf.get('failure-message', job.failure_message)
job.success_message = conf.get('success-message', job.success_message)

7
zuul/connection/__init__.py
View File

@ -62,3 +62,10 @@ class BaseConnection(object):
def registerUse(self, what, instance):
self.attached_to[what].append(instance)
def maintainCache(self, relevant):
"""Make cache contain relevant changes.
This lets the user supply a list of change objects that are
still in use. Anything in our cache that isn't in the supplied
list should be safe to remove from the cache."""

1
zuul/layoutvalidator.py
View File

@ -128,6 +128,7 @@ class LayoutSchema(object):
'hold-following-changes': bool,
'voting': bool,
'mutex': str,
'tags': toList(str),
'parameter-function': str,
'branch': toList(str),
'files': toList(str),

1
zuul/model.py
View File

@ -467,6 +467,7 @@ class Job(object):
swift=None, # TODOv3(jeblair): move to auth
parameter_function=None, # TODOv3(jeblair): remove
success_pattern=None, # TODOv3(jeblair): remove
tags=set(),
mutex=None,
)

15
zuul/scheduler.py
View File

@ -501,7 +501,7 @@ class Scheduler(threading.Thread):
"Exception while canceling build %s "
"for change %s" % (build, item.change))
# TODOv3(jeblair): update for tenants
self.maintainTriggerCache()
self.maintainConnectionCache()
for pipeline in tenant.layout.pipelines.values():
pipeline.source.postConfig()
pipeline.trigger.postConfig()
@ -626,19 +626,22 @@ class Scheduler(threading.Thread):
finally:
self.run_handler_lock.release()
def maintainTriggerCache(self):
def maintainConnectionCache(self):
# TODOv3(jeblair): update for tenants
relevant = set()
for tenant in self.abide.tenants.values():
for pipeline in tenant.layout.pipelines.values():
self.log.debug("Start maintain trigger cache for: %s" %
self.log.debug("Gather relevant cache items for: %s" %
pipeline)
for item in pipeline.getAllItems():
relevant.add(item.change)
relevant.update(item.change.getRelatedChanges())
pipeline.source.maintainCache(relevant)
self.log.debug("End maintain trigger cache for: %s" % pipeline)
self.log.debug("Trigger cache size: %s" % len(relevant))
for connection in self.connections.values():
connection.maintainCache(relevant)
self.log.debug(
"End maintain connection cache for: %s" % connection)
self.log.debug("Connection cache size: %s" % len(relevant))
def process_event_queue(self):
self.log.debug("Fetching trigger event")

7
zuul/source/__init__.py
View File

@ -46,13 +46,6 @@ class BaseSource(object):
def canMerge(self, change, allow_needs):
"""Determine if change can merge."""
def maintainCache(self, relevant):
"""Make cache contain relevant changes.
This lets the user supply a list of change objects that are
still in use. Anything in our cache that isn't in the supplied
list should be safe to remove from the cache."""
def postConfig(self):
"""Called after configuration has been processed."""

3
zuul/source/gerrit.py
View File

@ -320,6 +320,3 @@ class GerritSource(BaseSource):
def _getGitwebUrl(self, project, sha=None):
return self.connection.getGitwebUrl(project, sha)
def maintainCache(self, relevant):
self.connection.maintainCache(relevant)