zuul/zuul
Code Issues Proposed changes

Merge "Add --disable-pipelines option"

Browse Source
This commit is contained in:
Zuul
2024-05-20 20:06:40 +00:00
committed by Gerrit Code Review
parent 888015b16c 7028745bbe
commit 0489801751
7 changed files with 96 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
doc/source/operation.rst
View File

@ -65,6 +65,32 @@ changes to the configuration stored in ZooKeeper and automatically
update their configuration in the background without interrupting
processing.
Advanced Options
~~~~~~~~~~~~~~~~
These options are not necessary under normal conditions, but may be
useful in some complex environments.
The ``--wait-for-init`` option (or ``ZUUL_WAIT_FOR_INIT`` environment
variable) will cause the scheduler to wait until all tenants
have been initialized before it begins processing pipelines. This may
help large systems with excess scheduler capacity perform a rolling
restart of schedulers more quickly.
The ``--disable-pipelines`` option (or ``ZUUL_DISABLE_PIPELINES``
environment variable) will cause the scheduler to silently discard all
pipeline related events. This allows the scheduler to create and
maintain all of the configuration of a running system without running
any jobs or making any reports.
This option is not intended for normal use, and is only useful for
certain testing and backup-related activities. Because any scheduler
connected to ZooKeeper can process events, it does not make sense to
mix values of this option. Normal production Zuul systems that are
intended to process events should not set this option on any
schedulers. To use this option on a standby or testing cluster, set
it on all schedulers.
.. _backup:
Backup and Restoration

7
releasenotes/notes/disable-pipelines-c2e821d8ae8325d8.yaml Normal file
View File

@ -0,0 +1,7 @@
---
features:
- |
The scheduler now accepts the argument ``--disable-pipelines``
which will instruct it to discard all pipeline events. This
facilitaties the creation of a system with a running Zuul config
that does not start any jobs or make any reports.

24
tests/base.py
View File

@ -1850,12 +1850,13 @@ class SchedulerTestApp:
def __init__(self, log, config, changes, additional_event_queues,
upstream_root, poller_events,
git_url_with_auth, add_cleanup, validate_tenants,
wait_for_init, instance_id):
wait_for_init, disable_pipelines, instance_id):
self.log = log
self.config = config
self.changes = changes
self.validate_tenants = validate_tenants
self.wait_for_init = wait_for_init
self.disable_pipelines = disable_pipelines
# Register connections from the config using fakes
self.connections = TestConnectionRegistry(
@ -1870,7 +1871,7 @@ class SchedulerTestApp:
self.connections.configure(self.config)
self.sched = TestScheduler(self.config, self.connections, self,
wait_for_init)
wait_for_init, disable_pipelines)
self.sched.log = logging.getLogger(f"zuul.Scheduler-{instance_id}")
self.sched._stats_interval = 1
@ -1936,12 +1937,13 @@ class SchedulerTestApp:
class SchedulerTestManager:
def __init__(self, validate_tenants, wait_for_init):
def __init__(self, validate_tenants, wait_for_init, disable_pipelines):
self.instances = []
def create(self, log, config, changes, additional_event_queues,
upstream_root, poller_events, git_url_with_auth,
add_cleanup, validate_tenants, wait_for_init):
add_cleanup, validate_tenants, wait_for_init,
disable_pipelines):
# Since the config contains a regex we cannot use copy.deepcopy()
# as this will raise an exception with Python <3.7
config_data = StringIO()
@ -1960,9 +1962,10 @@ class SchedulerTestManager:
app = SchedulerTestApp(log, scheduler_config, changes,
additional_event_queues, upstream_root,
poller_events,
git_url_with_auth, add_cleanup,
validate_tenants, wait_for_init, instance_id)
poller_events, git_url_with_auth,
add_cleanup, validate_tenants,
wait_for_init, disable_pipelines,
instance_id)
self.instances.append(app)
return app
@ -2067,6 +2070,7 @@ class ZuulTestCase(BaseTestCase):
log_console_port: int = 19885
validate_tenants = None
wait_for_init = None
disable_pipelines = False
scheduler_count = SCHEDULER_COUNT
init_repos = True
load_change_db = False
@ -2239,7 +2243,8 @@ class ZuulTestCase(BaseTestCase):
self.builds = self.executor_server.running_builds
self.scheds = SchedulerTestManager(self.validate_tenants,
self.wait_for_init)
self.wait_for_init,
self.disable_pipelines)
for _ in range(self.scheduler_count):
self.createScheduler()
@ -2258,7 +2263,8 @@ class ZuulTestCase(BaseTestCase):
self.log, self.config, self.changes,
self.additional_event_queues, self.upstream_root,
self.poller_events, self.git_url_with_auth,
self.addCleanup, self.validate_tenants, self.wait_for_init)
self.addCleanup, self.validate_tenants, self.wait_for_init,
self.disable_pipelines)
def createZKContext(self, lock=None):
if lock is None:

17
tests/unit/test_scheduler.py
View File

@ -9489,3 +9489,20 @@ class TestWaitForInit(ZuulTestCase):
A.addApproval('Code-Review', 2)
self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
self.waitUntilSettled()
self.assertHistory([
dict(name='project-merge', result='SUCCESS', changes='1,1'),
dict(name='project-test1', result='SUCCESS', changes='1,1'),
dict(name='project-test2', result='SUCCESS', changes='1,1'),
], ordered=False)
class TestDisablePipelines(ZuulTestCase):
tenant_config_file = 'config/single-tenant/main.yaml'
disable_pipelines = True
def test_disable_pipelines(self):
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
A.addApproval('Code-Review', 2)
self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
self.waitUntilSettled()
self.assertHistory([])

10
zuul/cmd/__init__.py
View File

@ -162,6 +162,16 @@ class ZuulApp(object):
return parser
def envBool(self, name, default=None):
"""Get the named variable from the environment and
convert to boolean"""
val = os.getenv(name)
if val is None:
return default
if val.lower() in ['false', '0']:
return False
return True
def readConfig(self):
safe_env = {
k: v for k, v in os.environ.items()

12
zuul/cmd/scheduler.py
View File

@ -41,8 +41,15 @@ class Scheduler(zuul.cmd.ZuulDaemonApp):
'will distribute work to mergers.')
parser.add_argument('--wait-for-init', dest='wait_for_init',
action='store_true',
default=self.envBool('ZUUL_WAIT_FOR_INIT'),
help='Wait until all tenants are fully loaded '
'before beginning to process events.')
'before beginning to process events. '
'(also available as ZUUL_WAIT_FOR_INIT).')
parser.add_argument('--disable-pipelines', dest='disable_pipelines',
action='store_true',
default=self.envBool('ZUUL_DISABLE_PIPELINES'),
help='Discard all pipeline related events '
'(also available as ZUUL_DISABLE_PIPELINES).')
self.addSubCommands(parser, zuul.scheduler.COMMANDS)
return parser
@ -86,7 +93,8 @@ class Scheduler(zuul.cmd.ZuulDaemonApp):
self.configure_connections(require_sql=True)
self.sched = zuul.scheduler.Scheduler(self.config,
self.connections, self,
self.args.wait_for_init)
self.args.wait_for_init,
self.args.disable_pipelines)
if self.args.validate_tenants is None:
self.connections.registerScheduler(self.sched)
self.connections.load(self.sched.zk_client,

18
zuul/scheduler.py
View File

@ -211,9 +211,10 @@ class Scheduler(threading.Thread):
_executor_client_class = ExecutorClient
def __init__(self, config, connections, app, wait_for_init,
testonly=False):
disable_pipelines=False, testonly=False):
threading.Thread.__init__(self)
self.daemon = True
self.disable_pipelines = disable_pipelines
self._profile_pipelines = set()
self.wait_for_init = wait_for_init
self.hostname = socket.getfqdn()
@ -2589,10 +2590,11 @@ class Scheduler(threading.Thread):
return
log.debug("Processing trigger event %s", event)
try:
if isinstance(event, SupercedeEvent):
self._doSupercedeEvent(event)
else:
self._process_trigger_event(tenant, pipeline, event)
if not self.disable_pipelines:
if isinstance(event, SupercedeEvent):
self._doSupercedeEvent(event)
else:
self._process_trigger_event(tenant, pipeline, event)
finally:
self.pipeline_trigger_events[tenant.name][
pipeline.name
@ -2709,7 +2711,8 @@ class Scheduler(threading.Thread):
log = get_annotated_logger(self.log, event.zuul_event_id)
log.debug("Processing management event %s", event)
try:
self._process_management_event(event)
if not self.disable_pipelines:
self._process_management_event(event)
finally:
self.pipeline_management_events[tenant.name][
pipeline.name
@ -2751,7 +2754,8 @@ class Scheduler(threading.Thread):
)
log.debug("Processing result event %s", event)
try:
self._process_result_event(event, pipeline)
if not self.disable_pipelines:
self._process_result_event(event, pipeline)
finally:
self.pipeline_result_events[tenant.name][
pipeline.name