From e9d45c3c13962c8fb8439b2d05f76a66674dea74 Mon Sep 17 00:00:00 2001 From: "James E. Blair" Date: Thu, 31 May 2012 09:56:45 -0700 Subject: [PATCH] Add dynamic reconfiguration. When SIGHUP is received, trigger events are queued only, we wait for all builds to complete, re-load the configuration, then continue. Initial configuration is now performed the same way, to make sure it gets exercised. Change-Id: I41198b6dc9f176c8e57cd4a10ad00e4b7480e1d1 --- zuul-server | 94 ++++++++++++++++++++++++++++------------------- zuul/scheduler.py | 58 +++++++++++++++++++++++++---- 2 files changed, 107 insertions(+), 45 deletions(-) diff --git a/zuul-server b/zuul-server index bb7a1e6ab8..8d99e105dc 100755 --- a/zuul-server +++ b/zuul-server @@ -15,58 +15,78 @@ import argparse import ConfigParser +import logging.config import os +import signal import zuul.scheduler import zuul.launcher.jenkins import zuul.trigger.gerrit -import logging.config +class Server(object): + def __init__(self): + self.args = None + self.config = None -def parse_arguments(): - parser = argparse.ArgumentParser(description='Project gating system.') - parser.add_argument('-c', dest='config', - help='specify the config file') - return parser.parse_args() + def parse_arguments(self): + parser = argparse.ArgumentParser(description='Project gating system.') + parser.add_argument('-c', dest='config', + help='specify the config file') + self.args = parser.parse_args() + def read_config(self): + self.config = ConfigParser.ConfigParser() + if self.args.config: + locations = [self.args.config] + else: + locations = ['/etc/zuul/zuul.conf', + '~/zuul.conf'] + for fp in locations: + if os.path.exists(os.path.expanduser(fp)): + self.config.read(os.path.expanduser(fp)) + return + raise Exception("Unable to locate config file in %s" % locations) -def read_config(args): - config = ConfigParser.ConfigParser() - if args.config: - locations = [args.config] - else: - locations = ['/etc/zuul/zuul.conf', - '~/zuul.conf'] - for fp in locations: - if os.path.exists(os.path.expanduser(fp)): - config.read(fp) - return config - raise Exception("Unable to locate config file in %s" % locations) + def setup_logging(self): + if self.config.has_option('zuul', 'log_config'): + fp = os.path.expanduser(self.config.get('zuul', 'log_config')) + if not os.path.exists(fp): + raise Exception("Unable to read logging config file at %s" % + fp) + logging.config.fileConfig(fp) + else: + logging.basicConfig(level=logging.DEBUG) + def reconfigure_handler(self, signum, frame): + signal.signal(signal.SIGHUP, signal.SIG_IGN) + self.read_config() + self.setup_logging() + self.sched.reconfigure(self.config) + signal.signal(signal.SIGHUP, self.reconfigure_handler) -def setup_logging(config): - if config.has_option('zuul', 'log_config'): - fp = os.path.expanduser(config.get('zuul', 'log_config')) - if not os.path.exists(fp): - raise Exception("Unable to read logging config file at %s" % fp) - logging.config.fileConfig(fp) - else: - logging.basicConfig(level=logging.DEBUG) + def main(self): + self.sched = zuul.scheduler.Scheduler() + jenkins = zuul.launcher.jenkins.Jenkins(self.config, self.sched) + gerrit = zuul.trigger.gerrit.Gerrit(self.config, self.sched) -def main(config): - sched = zuul.scheduler.Scheduler(config) + self.sched.setLauncher(jenkins) + self.sched.setTrigger(gerrit) - jenkins = zuul.launcher.jenkins.Jenkins(config, sched) - gerrit = zuul.trigger.gerrit.Gerrit(config, sched) + self.sched.start() + self.sched.reconfigure(self.config) + signal.signal(signal.SIGHUP, self.reconfigure_handler) + while True: + signal.pause() + + def start(self): + self.parse_arguments() + self.read_config() + self.setup_logging() + self.main() - sched.setLauncher(jenkins) - sched.setTrigger(gerrit) - sched.run() if __name__ == '__main__': - args = parse_arguments() - config = read_config(args) - setup_logging(config) - main(config) + server = Server() + server.start() diff --git a/zuul/scheduler.py b/zuul/scheduler.py index 4084bb5557..acc5a88edb 100644 --- a/zuul/scheduler.py +++ b/zuul/scheduler.py @@ -21,21 +21,24 @@ import yaml from model import Job, Change, Project, ChangeQueue, EventFilter -class Scheduler(object): +class Scheduler(threading.Thread): log = logging.getLogger("zuul.Scheduler") - def __init__(self, config): + def __init__(self): + threading.Thread.__init__(self) self.wake_event = threading.Event() - self.queue_managers = {} - self.jobs = {} - self.projects = {} + self.reconfigure_complete_event = threading.Event() self.launcher = None self.trigger = None self.trigger_event_queue = Queue.Queue() self.result_event_queue = Queue.Queue() + self._init() - self._parseConfig(config.get('zuul', 'layout_config')) + def _init(self): + self.queue_managers = {} + self.jobs = {} + self.projects = {} def _parseConfig(self, fp): def toList(item): @@ -130,6 +133,36 @@ class Scheduler(object): self.result_event_queue.put(build) self.wake_event.set() + def reconfigure(self, config): + self.log.debug("Reconfigure") + self.config = config + self._reconfigure_flag = True + self.wake_event.set() + self.log.debug("Waiting for reconfiguration") + self.reconfigure_complete_event.wait() + self.reconfigure_complete_event.clear() + self.log.debug("Reconfiguration complete") + + def _doReconfigure(self): + self.log.debug("Performing reconfiguration") + self._init() + self._parseConfig(self.config.get('zuul', 'layout_config')) + self._reconfigure_flag = False + self.reconfigure_complete_event.set() + + def _areAllBuildsComplete(self): + self.log.debug("Checking if all builds are complete") + waiting = False + for manager in self.queue_managers.values(): + for build in manager.building_jobs.values(): + self.log.debug("%s waiting on %s" % (manager, build)) + waiting = True + if not waiting: + self.log.debug("All builds are complete") + return True + self.log.debug("All builds are not complete") + return False + def run(self): while True: self.log.debug("Run handler sleeping") @@ -137,10 +170,19 @@ class Scheduler(object): self.wake_event.clear() self.log.debug("Run handler awake") try: - if not self.trigger_event_queue.empty(): - self.process_event_queue() + if not self._reconfigure_flag: + if not self.trigger_event_queue.empty(): + self.process_event_queue() + if not self.result_event_queue.empty(): self.process_result_queue() + + if self._reconfigure_flag and self._areAllBuildsComplete(): + self._doReconfigure() + + if not (self.trigger_event_queue.empty() and + self.result_event_queue.empty()): + self.wake_event.set() except: self.log.exception("Exception in run handler:")