James E. Blair
a77f28c887
Gear 0.5.0 added statsd support for the gear server. Have Zuul use it based on the existence of the same env vars Zuul uses to configure its own use of statsd. Change-Id: I09ea2541d1f2eff5f4fd2e0c78da2acfdc1776ab
275 lines
9.8 KiB
Python
Executable File
275 lines
9.8 KiB
Python
Executable File
#!/usr/bin/env python
|
|
# Copyright 2012 Hewlett-Packard Development Company, L.P.
|
|
# Copyright 2013 OpenStack Foundation
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
# not use this file except in compliance with the License. You may obtain
|
|
# a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
import argparse
|
|
import ConfigParser
|
|
import daemon
|
|
import extras
|
|
|
|
# as of python-daemon 1.6 it doesn't bundle pidlockfile anymore
|
|
# instead it depends on lockfile-0.9.1 which uses pidfile.
|
|
pid_file_module = extras.try_imports(['daemon.pidlockfile', 'daemon.pidfile'])
|
|
|
|
import logging
|
|
import logging.config
|
|
import os
|
|
import sys
|
|
import signal
|
|
import traceback
|
|
|
|
# No zuul imports here because they pull in paramiko which must not be
|
|
# imported until after the daemonization.
|
|
# https://github.com/paramiko/paramiko/issues/59
|
|
# Similar situation with gear and statsd.
|
|
|
|
|
|
def stack_dump_handler(signum, frame):
|
|
signal.signal(signal.SIGUSR2, signal.SIG_IGN)
|
|
log_str = ""
|
|
for thread_id, stack_frame in sys._current_frames().items():
|
|
log_str += "Thread: %s\n" % thread_id
|
|
log_str += "".join(traceback.format_stack(stack_frame))
|
|
log = logging.getLogger("zuul.stack_dump")
|
|
log.debug(log_str)
|
|
signal.signal(signal.SIGUSR2, stack_dump_handler)
|
|
|
|
|
|
class Server(object):
|
|
def __init__(self):
|
|
self.args = None
|
|
self.config = None
|
|
self.gear_server_pid = None
|
|
|
|
def parse_arguments(self):
|
|
parser = argparse.ArgumentParser(description='Project gating system.')
|
|
parser.add_argument('-c', dest='config',
|
|
help='specify the config file')
|
|
parser.add_argument('-l', dest='layout',
|
|
help='specify the layout file')
|
|
parser.add_argument('-d', dest='nodaemon', action='store_true',
|
|
help='do not run as a daemon')
|
|
parser.add_argument('-t', dest='validate', nargs='?', const=True,
|
|
metavar='JOB_LIST',
|
|
help='validate layout file syntax (optionally '
|
|
'providing the path to a file with a list of '
|
|
'available job names)')
|
|
parser.add_argument('--version', dest='version', action='version',
|
|
version=self._get_version(),
|
|
help='show zuul version')
|
|
self.args = parser.parse_args()
|
|
|
|
def _get_version(self):
|
|
from zuul.version import version_info as zuul_version_info
|
|
return "Zuul version: %s" % zuul_version_info.version_string()
|
|
|
|
def read_config(self):
|
|
self.config = ConfigParser.ConfigParser()
|
|
if self.args.config:
|
|
locations = [self.args.config]
|
|
else:
|
|
locations = ['/etc/zuul/zuul.conf',
|
|
'~/zuul.conf']
|
|
for fp in locations:
|
|
if os.path.exists(os.path.expanduser(fp)):
|
|
self.config.read(os.path.expanduser(fp))
|
|
return
|
|
raise Exception("Unable to locate config file in %s" % locations)
|
|
|
|
def setup_logging(self, section, parameter):
|
|
if self.config.has_option(section, parameter):
|
|
fp = os.path.expanduser(self.config.get(section, parameter))
|
|
if not os.path.exists(fp):
|
|
raise Exception("Unable to read logging config file at %s" %
|
|
fp)
|
|
logging.config.fileConfig(fp)
|
|
else:
|
|
logging.basicConfig(level=logging.DEBUG)
|
|
|
|
def reconfigure_handler(self, signum, frame):
|
|
signal.signal(signal.SIGHUP, signal.SIG_IGN)
|
|
self.read_config()
|
|
self.setup_logging('zuul', 'log_config')
|
|
self.sched.reconfigure(self.config)
|
|
signal.signal(signal.SIGHUP, self.reconfigure_handler)
|
|
|
|
def exit_handler(self, signum, frame):
|
|
signal.signal(signal.SIGUSR1, signal.SIG_IGN)
|
|
self.sched.exit()
|
|
self.sched.join()
|
|
self.stop_gear_server()
|
|
|
|
def term_handler(self, signum, frame):
|
|
self.stop_gear_server()
|
|
os._exit(0)
|
|
|
|
def test_config(self, job_list_path):
|
|
# See comment at top of file about zuul imports
|
|
import zuul.scheduler
|
|
import zuul.launcher.gearman
|
|
import zuul.trigger.gerrit
|
|
|
|
logging.basicConfig(level=logging.DEBUG)
|
|
self.sched = zuul.scheduler.Scheduler()
|
|
self.sched.registerReporter(None, 'gerrit')
|
|
self.sched.registerReporter(None, 'smtp')
|
|
self.sched.registerTrigger(None, 'gerrit')
|
|
self.sched.registerTrigger(None, 'timer')
|
|
layout = self.sched.testConfig(self.config.get('zuul',
|
|
'layout_config'))
|
|
if not job_list_path:
|
|
return False
|
|
|
|
failure = False
|
|
path = os.path.expanduser(job_list_path)
|
|
if not os.path.exists(path):
|
|
raise Exception("Unable to find job list: %s" % path)
|
|
jobs = set()
|
|
for line in open(path):
|
|
v = line.strip()
|
|
if v:
|
|
jobs.add(v)
|
|
for job in sorted(layout.jobs):
|
|
if job not in jobs:
|
|
print "Job %s not defined" % job
|
|
failure = True
|
|
return failure
|
|
|
|
def start_gear_server(self):
|
|
pipe_read, pipe_write = os.pipe()
|
|
child_pid = os.fork()
|
|
if child_pid == 0:
|
|
os.close(pipe_write)
|
|
self.setup_logging('gearman_server', 'log_config')
|
|
import gear
|
|
statsd_host = os.environ.get('STATSD_HOST')
|
|
statsd_port = int(os.environ.get('STATSD_PORT', 8125))
|
|
gear.Server(4730,
|
|
statsd_host=statsd_host,
|
|
statsd_port=statsd_port,
|
|
statsd_prefix='zuul.geard')
|
|
|
|
# Keep running until the parent dies:
|
|
pipe_read = os.fdopen(pipe_read)
|
|
pipe_read.read()
|
|
os._exit(0)
|
|
else:
|
|
os.close(pipe_read)
|
|
self.gear_server_pid = child_pid
|
|
self.gear_pipe_write = pipe_write
|
|
|
|
def stop_gear_server(self):
|
|
if self.gear_server_pid:
|
|
os.kill(self.gear_server_pid, signal.SIGKILL)
|
|
|
|
def main(self):
|
|
# See comment at top of file about zuul imports
|
|
import zuul.scheduler
|
|
import zuul.launcher.gearman
|
|
import zuul.merger.client
|
|
import zuul.reporter.gerrit
|
|
import zuul.reporter.smtp
|
|
import zuul.trigger.gerrit
|
|
import zuul.trigger.timer
|
|
import zuul.webapp
|
|
import zuul.rpclistener
|
|
|
|
if (self.config.has_option('gearman_server', 'start') and
|
|
self.config.getboolean('gearman_server', 'start')):
|
|
self.start_gear_server()
|
|
|
|
self.setup_logging('zuul', 'log_config')
|
|
self.log = logging.getLogger("zuul.Server")
|
|
|
|
self.sched = zuul.scheduler.Scheduler()
|
|
|
|
gearman = zuul.launcher.gearman.Gearman(self.config, self.sched)
|
|
merger = zuul.merger.client.MergeClient(self.config, self.sched)
|
|
gerrit = zuul.trigger.gerrit.Gerrit(self.config, self.sched)
|
|
timer = zuul.trigger.timer.Timer(self.config, self.sched)
|
|
webapp = zuul.webapp.WebApp(self.sched)
|
|
rpc = zuul.rpclistener.RPCListener(self.config, self.sched)
|
|
gerrit_reporter = zuul.reporter.gerrit.Reporter(gerrit)
|
|
smtp_reporter = zuul.reporter.smtp.Reporter(
|
|
self.config.get('smtp', 'default_from')
|
|
if self.config.has_option('smtp', 'default_from') else 'zuul',
|
|
self.config.get('smtp', 'default_to')
|
|
if self.config.has_option('smtp', 'default_to') else 'zuul',
|
|
self.config.get('smtp', 'server')
|
|
if self.config.has_option('smtp', 'server') else 'localhost',
|
|
self.config.get('smtp', 'port')
|
|
if self.config.has_option('smtp', 'port') else 25
|
|
)
|
|
|
|
self.sched.setLauncher(gearman)
|
|
self.sched.setMerger(merger)
|
|
self.sched.registerTrigger(gerrit)
|
|
self.sched.registerTrigger(timer)
|
|
self.sched.registerReporter(gerrit_reporter)
|
|
self.sched.registerReporter(smtp_reporter)
|
|
|
|
self.log.info('Starting scheduler')
|
|
self.sched.start()
|
|
self.sched.reconfigure(self.config)
|
|
self.sched.resume()
|
|
self.log.info('Starting Webapp')
|
|
webapp.start()
|
|
self.log.info('Starting RPC')
|
|
rpc.start()
|
|
|
|
signal.signal(signal.SIGHUP, self.reconfigure_handler)
|
|
signal.signal(signal.SIGUSR1, self.exit_handler)
|
|
signal.signal(signal.SIGUSR2, stack_dump_handler)
|
|
signal.signal(signal.SIGTERM, self.term_handler)
|
|
while True:
|
|
try:
|
|
signal.pause()
|
|
except KeyboardInterrupt:
|
|
print "Ctrl + C: asking scheduler to exit nicely...\n"
|
|
self.exit_handler(signal.SIGINT, None)
|
|
|
|
|
|
def main():
|
|
server = Server()
|
|
server.parse_arguments()
|
|
|
|
server.read_config()
|
|
|
|
if server.args.layout:
|
|
server.config.set('zuul', 'layout_config', server.args.layout)
|
|
|
|
if server.args.validate:
|
|
path = server.args.validate
|
|
if path is True:
|
|
path = None
|
|
sys.exit(server.test_config(path))
|
|
|
|
if server.config.has_option('zuul', 'pidfile'):
|
|
pid_fn = os.path.expanduser(server.config.get('zuul', 'pidfile'))
|
|
else:
|
|
pid_fn = '/var/run/zuul/zuul.pid'
|
|
pid = pid_file_module.TimeoutPIDLockFile(pid_fn, 10)
|
|
|
|
if server.args.nodaemon:
|
|
server.main()
|
|
else:
|
|
with daemon.DaemonContext(pidfile=pid):
|
|
server.main()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.path.insert(0, '.')
|
|
main()
|