zuul
/
zuul
Code Issues Proposed changes

Merge "Normalize daemon process handling" into feature/zuulv3

This commit is contained in:
Zuul 2017-11-29 21:47:57 +00:00 committed by Gerrit Code Review
parent 6c4325cbcf de0248e0d5
commit 942d2d6acc
7 changed files with 109 additions and 225 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
tests/unit/test_scheduler_cmd.py
View File

@ -1,34 +0,0 @@
#!/usr/bin/env python
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import os
import testtools
import zuul.cmd.scheduler
from tests import base
class TestSchedulerCmdArguments(testtools.TestCase):
def setUp(self):
super(TestSchedulerCmdArguments, self).setUp()
self.app = zuul.cmd.scheduler.Scheduler()
def test_test_config(self):
conf_path = os.path.join(base.FIXTURE_DIR, 'zuul.conf')
self.app.parse_arguments(['-t', '-c', conf_path])
self.assertTrue(self.app.args.validate)
self.app.read_config()
self.assertEqual(0, self.app.test_config())

56
zuul/cmd/__init__.py
View File

@ -14,7 +14,9 @@
# License for the specific language governing permissions and limitations
# under the License.
import argparse
import configparser
import daemon
import extras
import io
import logging
@ -28,8 +30,13 @@ import threading
yappi = extras.try_import('yappi')
objgraph = extras.try_import('objgraph')
# as of python-daemon 1.6 it doesn't bundle pidlockfile anymore
# instead it depends on lockfile-0.9.1 which uses pidfile.
pid_file_module = extras.try_imports(['daemon.pidlockfile', 'daemon.pidfile'])
from zuul.ansible import logconfig
import zuul.lib.connections
from zuul.lib.config import get_default
# Do not import modules that will pull in paramiko which must not be
# imported until after the daemonization.
@ -87,6 +94,8 @@ def stack_dump_handler(signum, frame):
class ZuulApp(object):
app_name = None # type: str
app_description = None # type: str
def __init__(self):
self.args = None
@ -97,7 +106,21 @@ class ZuulApp(object):
from zuul.version import version_info as zuul_version_info
return "Zuul version: %s" % zuul_version_info.release_string()
def read_config(self):
def createParser(self):
parser = argparse.ArgumentParser(description=self.app_description)
parser.add_argument('-c', dest='config',
help='specify the config file')
parser.add_argument('--version', dest='version', action='version',
version=self._get_version(),
help='show zuul version')
return parser
def parseArguments(self, args=None):
parser = self.createParser()
self.args = parser.parse_args(args)
return parser
def readConfig(self):
self.config = configparser.ConfigParser()
if self.args.config:
locations = [self.args.config]
@ -130,3 +153,34 @@ class ZuulApp(object):
def configure_connections(self, source_only=False):
self.connections = zuul.lib.connections.ConnectionRegistry()
self.connections.configure(self.config, source_only)
class ZuulDaemonApp(ZuulApp):
def createParser(self):
parser = super(ZuulDaemonApp, self).createParser()
parser.add_argument('-d', dest='nodaemon', action='store_true',
help='do not run as a daemon')
return parser
def getPidFile(self):
pid_fn = get_default(self.config, self.app_name, 'pidfile',
'/var/run/zuul/%s.pid' % self.app_name,
expand_user=True)
return pid_fn
def main(self):
self.parseArguments()
self.readConfig()
pid_fn = self.getPidFile()
pid = pid_file_module.TimeoutPIDLockFile(pid_fn, 10)
if self.args.nodaemon:
self.run()
else:
# Exercise the pidfile before we do anything else (including
# logging or daemonizing)
with daemon.DaemonContext(pidfile=pid):
pass
with daemon.DaemonContext(pidfile=pid):
self.run()

25
zuul/cmd/client.py
View File

@ -30,18 +30,14 @@ from zuul.lib.config import get_default
class Client(zuul.cmd.ZuulApp):
app_name = 'zuul'
app_description = 'Zuul RPC client.'
log = logging.getLogger("zuul.Client")
def parse_arguments(self):
parser = argparse.ArgumentParser(
description='Zuul Project Gating System Client.')
parser.add_argument('-c', dest='config',
help='specify the config file')
def createParser(self):
parser = super(Client, self).createParser()
parser.add_argument('-v', dest='verbose', action='store_true',
help='verbose output')
parser.add_argument('--version', dest='version', action='version',
version=self._get_version(),
help='show zuul version')
subparsers = parser.add_subparsers(title='commands',
description='valid commands',
@ -133,7 +129,10 @@ class Client(zuul.cmd.ZuulApp):
# TODO: add filters such as queue, project, changeid etc
show_running_jobs.set_defaults(func=self.show_running_jobs)
self.args = parser.parse_args()
return parser
def parseArguments(self, args=None):
parser = super(Client, self).parseArguments()
if not getattr(self.args, 'func', None):
parser.print_help()
sys.exit(1)
@ -156,8 +155,8 @@ class Client(zuul.cmd.ZuulApp):
logging.basicConfig(level=logging.DEBUG)
def main(self):
self.parse_arguments()
self.read_config()
self.parseArguments()
self.readConfig()
self.setup_logging()
self.server = self.config.get('gearman', 'server')
@ -363,10 +362,8 @@ class Client(zuul.cmd.ZuulApp):
def main():
client = Client()
client.main()
Client().main()
if __name__ == "__main__":
sys.path.insert(0, '.')
main()

66
zuul/cmd/executor.py
View File

@ -14,14 +14,6 @@
# License for the specific language governing permissions and limitations
# under the License.
import argparse
import daemon
import extras
# as of python-daemon 1.6 it doesn't bundle pidlockfile anymore
# instead it depends on lockfile-0.9.1 which uses pidfile.
pid_file_module = extras.try_imports(['daemon.pidlockfile', 'daemon.pidfile'])
import grp
import logging
import os
@ -41,25 +33,24 @@ from zuul.lib.config import get_default
# Similar situation with gear and statsd.
class Executor(zuul.cmd.ZuulApp):
class Executor(zuul.cmd.ZuulDaemonApp):
app_name = 'executor'
app_description = 'A standalone Zuul executor.'
def parse_arguments(self):
parser = argparse.ArgumentParser(description='Zuul executor.')
parser.add_argument('-c', dest='config',
help='specify the config file')
parser.add_argument('-d', dest='nodaemon', action='store_true',
help='do not run as a daemon')
parser.add_argument('--version', dest='version', action='version',
version=self._get_version(),
help='show zuul version')
def createParser(self):
parser = super(Executor, self).createParser()
parser.add_argument('--keep-jobdir', dest='keep_jobdir',
action='store_true',
help='keep local jobdirs after run completes')
parser.add_argument('command',
choices=zuul.executor.server.COMMANDS,
nargs='?')
return parser
self.args = parser.parse_args()
def parseArguments(self, args=None):
super(Executor, self).parseArguments()
if self.args.command:
self.args.nodaemon = True
def send_command(self, cmd):
state_dir = get_default(self.config, 'executor', 'state_dir',
@ -111,8 +102,12 @@ class Executor(zuul.cmd.ZuulApp):
os.chdir(pw.pw_dir)
os.umask(0o022)
def main(self, daemon=True):
# See comment at top of file about zuul imports
def run(self):
if self.args.command in zuul.executor.server.COMMANDS:
self.send_command(self.args.command)
sys.exit(0)
self.configure_connections(source_only=True)
self.user = get_default(self.config, 'executor', 'user', 'zuul')
@ -145,9 +140,8 @@ class Executor(zuul.cmd.ZuulApp):
self.executor.start()
signal.signal(signal.SIGUSR2, zuul.cmd.stack_dump_handler)
if daemon:
self.executor.join()
else:
if self.args.nodaemon:
while True:
try:
signal.pause()
@ -155,31 +149,13 @@ class Executor(zuul.cmd.ZuulApp):
print("Ctrl + C: asking executor to exit nicely...\n")
self.exit_handler()
sys.exit(0)
else:
self.executor.join()
def main():
server = Executor()
server.parse_arguments()
server.read_config()
if server.args.command in zuul.executor.server.COMMANDS:
server.send_command(server.args.command)
sys.exit(0)
server.configure_connections(source_only=True)
pid_fn = get_default(server.config, 'executor', 'pidfile',
'/var/run/zuul-executor/zuul-executor.pid',
expand_user=True)
pid = pid_file_module.TimeoutPIDLockFile(pid_fn, 10)
if server.args.nodaemon:
server.main(False)
else:
with daemon.DaemonContext(pidfile=pid):
server.main(True)
Executor().main()
if __name__ == "__main__":
sys.path.insert(0, '.')
main()

47
zuul/cmd/merger.py
View File

@ -14,19 +14,9 @@
# License for the specific language governing permissions and limitations
# under the License.
import argparse
import daemon
import extras
# as of python-daemon 1.6 it doesn't bundle pidlockfile anymore
# instead it depends on lockfile-0.9.1 which uses pidfile.
pid_file_module = extras.try_imports(['daemon.pidlockfile', 'daemon.pidfile'])
import sys
import signal
import zuul.cmd
from zuul.lib.config import get_default
# No zuul imports here because they pull in paramiko which must not be
# imported until after the daemonization.
@ -34,28 +24,21 @@ from zuul.lib.config import get_default
# Similar situation with gear and statsd.
class Merger(zuul.cmd.ZuulApp):
def parse_arguments(self):
parser = argparse.ArgumentParser(description='Zuul merge worker.')
parser.add_argument('-c', dest='config',
help='specify the config file')
parser.add_argument('-d', dest='nodaemon', action='store_true',
help='do not run as a daemon')
parser.add_argument('--version', dest='version', action='version',
version=self._get_version(),
help='show zuul version')
self.args = parser.parse_args()
class Merger(zuul.cmd.ZuulDaemonApp):
app_name = 'merger'
app_description = 'A standalone Zuul merger.'
def exit_handler(self, signum, frame):
signal.signal(signal.SIGUSR1, signal.SIG_IGN)
self.merger.stop()
self.merger.join()
def main(self):
def run(self):
# See comment at top of file about zuul imports
import zuul.merger.server
self.configure_connections(source_only=True)
self.setup_logging('merger', 'log_config')
self.merger = zuul.merger.server.MergeServer(self.config,
@ -73,24 +56,8 @@ class Merger(zuul.cmd.ZuulApp):
def main():
server = Merger()
server.parse_arguments()
server.read_config()
server.configure_connections(source_only=True)
pid_fn = get_default(server.config, 'merger', 'pidfile',
'/var/run/zuul-merger/zuul-merger.pid',
expand_user=True)
pid = pid_file_module.TimeoutPIDLockFile(pid_fn, 10)
if server.args.nodaemon:
server.main()
else:
with daemon.DaemonContext(pidfile=pid):
server.main()
Merger().main()
if __name__ == "__main__":
sys.path.insert(0, '.')
main()

63
zuul/cmd/scheduler.py
View File

@ -14,14 +14,6 @@
# License for the specific language governing permissions and limitations
# under the License.
import argparse
import daemon
import extras
# as of python-daemon 1.6 it doesn't bundle pidlockfile anymore
# instead it depends on lockfile-0.9.1 which uses pidfile.
pid_file_module = extras.try_imports(['daemon.pidlockfile', 'daemon.pidfile'])
import logging
import os
import sys
@ -37,25 +29,14 @@ from zuul.lib.statsd import get_statsd_config
# Similar situation with gear and statsd.
class Scheduler(zuul.cmd.ZuulApp):
class Scheduler(zuul.cmd.ZuulDaemonApp):
app_name = 'scheduler'
app_description = 'The main zuul process.'
def __init__(self):
super(Scheduler, self).__init__()
self.gear_server_pid = None
def parse_arguments(self, args=None):
parser = argparse.ArgumentParser(description='Project gating system.')
parser.add_argument('-c', dest='config',
help='specify the config file')
parser.add_argument('-d', dest='nodaemon', action='store_true',
help='do not run as a daemon')
parser.add_argument('-t', dest='validate', action='store_true',
help='validate config file syntax (Does not'
'validate config repo validity)')
parser.add_argument('--version', dest='version', action='version',
version=self._get_version(),
help='show zuul version')
self.args = parser.parse_args(args)
def reconfigure_handler(self, signum, frame):
signal.signal(signal.SIGHUP, signal.SIG_IGN)
self.log.debug("Reconfiguration triggered")
@ -77,20 +58,6 @@ class Scheduler(zuul.cmd.ZuulApp):
self.stop_gear_server()
os._exit(0)
def test_config(self):
# See comment at top of file about zuul imports
import zuul.scheduler
import zuul.executor.client
logging.basicConfig(level=logging.DEBUG)
try:
self.sched = zuul.scheduler.Scheduler(self.config,
testonly=True)
except Exception as e:
self.log.error("%s" % e)
return -1
return 0
def start_gear_server(self):
pipe_read, pipe_write = os.pipe()
child_pid = os.fork()
@ -134,7 +101,7 @@ class Scheduler(zuul.cmd.ZuulApp):
if self.gear_server_pid:
os.kill(self.gear_server_pid, signal.SIGKILL)
def main(self):
def run(self):
# See comment at top of file about zuul imports
import zuul.scheduler
import zuul.executor.client
@ -206,26 +173,8 @@ class Scheduler(zuul.cmd.ZuulApp):
def main():
scheduler = Scheduler()
scheduler.parse_arguments()
scheduler.read_config()
if scheduler.args.validate:
sys.exit(scheduler.test_config())
pid_fn = get_default(scheduler.config, 'scheduler', 'pidfile',
'/var/run/zuul-scheduler/zuul-scheduler.pid',
expand_user=True)
pid = pid_file_module.TimeoutPIDLockFile(pid_fn, 10)
if scheduler.args.nodaemon:
scheduler.main()
else:
with daemon.DaemonContext(pidfile=pid):
scheduler.main()
Scheduler().main()
if __name__ == "__main__":
sys.path.insert(0, '.')
main()

43
zuul/cmd/web.py
View File

@ -13,10 +13,7 @@
# License for the specific language governing permissions and limitations
# under the License.
import argparse
import asyncio
import daemon
import extras
import logging
import signal
import sys
@ -27,28 +24,15 @@ import zuul.web
from zuul.lib.config import get_default
# as of python-daemon 1.6 it doesn't bundle pidlockfile anymore
# instead it depends on lockfile-0.9.1 which uses pidfile.
pid_file_module = extras.try_imports(['daemon.pidlockfile', 'daemon.pidfile'])
class WebServer(zuul.cmd.ZuulApp):
def parse_arguments(self):
parser = argparse.ArgumentParser(description='Zuul Web Server.')
parser.add_argument('-c', dest='config',
help='specify the config file')
parser.add_argument('-d', dest='nodaemon', action='store_true',
help='do not run as a daemon')
parser.add_argument('--version', dest='version', action='version',
version=self._get_version(),
help='show zuul version')
self.args = parser.parse_args()
class WebServer(zuul.cmd.ZuulDaemonApp):
app_name = 'web'
app_description = 'A standalone Zuul web server.'
def exit_handler(self, signum, frame):
self.web.stop()
def _main(self):
def _run(self):
params = dict()
params['listen_address'] = get_default(self.config,
@ -91,28 +75,19 @@ class WebServer(zuul.cmd.ZuulApp):
loop.close()
self.log.info("Zuul Web Server stopped")
def main(self):
def run(self):
self.setup_logging('web', 'log_config')
self.log = logging.getLogger("zuul.WebServer")
try:
self._main()
self._run()
except Exception:
self.log.exception("Exception from WebServer:")
def main():
server = WebServer()
server.parse_arguments()
server.read_config()
WebServer().main()
pid_fn = get_default(server.config, 'web', 'pidfile',
'/var/run/zuul-web/zuul-web.pid', expand_user=True)
pid = pid_file_module.TimeoutPIDLockFile(pid_fn, 10)
if server.args.nodaemon:
server.main()
else:
with daemon.DaemonContext(pidfile=pid):
server.main()
if __name__ == "__main__":
main()