c1b0a00c60
The reason for this is that containers for zuul services need to run privileged in order to successfully run bwrap. We currently only expect users to run the executor as privilged and the new bwrap execution checks have broken other services as a result. (Other services load the bwrap system bceause it is a normal zuul driver and all drivers are loaded by all services). This works around this by add a check_bwrap flag to connection setup and only setting it to true on the executor. A better longer term followup fixup would be to only instantiate the bwrap driver on the executor in the first place. This can probably be accomplished by overriding the ZuulApp configure_connections method in the executor and dropping bwrap creation in ZuulApp. Temporarily stop running the quick-start job since it's apparently not using speculative images. Change-Id: Ibadac0450e2879ef1ccc4b308ebd65de6e5a75ab
268 lines
9.5 KiB
Python
Executable File
268 lines
9.5 KiB
Python
Executable File
# Copyright 2012 Hewlett-Packard Development Company, L.P.
|
|
# Copyright 2013 OpenStack Foundation
|
|
# Copyright 2021-2022 Acme Gating, LLC
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
# not use this file except in compliance with the License. You may obtain
|
|
# a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
import abc
|
|
import argparse
|
|
import configparser
|
|
import daemon
|
|
import extras
|
|
import io
|
|
import json
|
|
import logging
|
|
import logging.config
|
|
import os
|
|
import signal
|
|
import socket
|
|
import sys
|
|
import traceback
|
|
import threading
|
|
|
|
yappi = extras.try_import('yappi')
|
|
objgraph = extras.try_import('objgraph')
|
|
|
|
# as of python-daemon 1.6 it doesn't bundle pidlockfile anymore
|
|
# instead it depends on lockfile-0.9.1 which uses pidfile.
|
|
pid_file_module = extras.try_imports(['daemon.pidlockfile', 'daemon.pidfile'])
|
|
|
|
from zuul.ansible import logconfig
|
|
import zuul.lib.connections
|
|
from zuul.lib.config import get_default
|
|
|
|
|
|
def stack_dump_handler(signum, frame):
|
|
signal.signal(signal.SIGUSR2, signal.SIG_IGN)
|
|
log = logging.getLogger("zuul.stack_dump")
|
|
log.debug("Beginning debug handler")
|
|
try:
|
|
threads = {}
|
|
for t in threading.enumerate():
|
|
threads[t.ident] = t
|
|
log_str = ""
|
|
for thread_id, stack_frame in sys._current_frames().items():
|
|
thread = threads.get(thread_id)
|
|
if thread:
|
|
thread_name = thread.name
|
|
thread_is_daemon = str(thread.daemon)
|
|
else:
|
|
thread_name = '(Unknown)'
|
|
thread_is_daemon = '(Unknown)'
|
|
log_str += "Thread: %s %s d: %s\n"\
|
|
% (thread_id, thread_name, thread_is_daemon)
|
|
log_str += "".join(traceback.format_stack(stack_frame))
|
|
log.debug(log_str)
|
|
except Exception:
|
|
log.exception("Thread dump error:")
|
|
try:
|
|
if yappi:
|
|
if not yappi.is_running():
|
|
log.debug("Starting Yappi")
|
|
yappi.start()
|
|
else:
|
|
log.debug("Stopping Yappi")
|
|
yappi.stop()
|
|
yappi_out = io.StringIO()
|
|
yappi.get_func_stats().print_all(out=yappi_out)
|
|
yappi.get_thread_stats().print_all(out=yappi_out)
|
|
log.debug(yappi_out.getvalue())
|
|
yappi_out.close()
|
|
yappi.clear_stats()
|
|
except Exception:
|
|
log.exception("Yappi error:")
|
|
try:
|
|
if objgraph:
|
|
log.debug("Most common types:")
|
|
objgraph_out = io.StringIO()
|
|
objgraph.show_growth(limit=100, file=objgraph_out)
|
|
log.debug(objgraph_out.getvalue())
|
|
objgraph_out.close()
|
|
except Exception:
|
|
log.exception("Objgraph error:")
|
|
log.debug("End debug handler")
|
|
signal.signal(signal.SIGUSR2, stack_dump_handler)
|
|
|
|
|
|
class ZuulApp(object):
|
|
app_name = None # type: str
|
|
app_description = None # type: str
|
|
|
|
def __init__(self):
|
|
self.args = None
|
|
self.config = None
|
|
self.connections = {}
|
|
self.commands = {}
|
|
|
|
def _get_version(self):
|
|
from zuul.version import release_string
|
|
return "Zuul version: %s" % release_string
|
|
|
|
def createParser(self):
|
|
parser = argparse.ArgumentParser(
|
|
description=self.app_description,
|
|
formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
parser.add_argument('-c', dest='config',
|
|
help='specify the config file')
|
|
parser.add_argument('--version', dest='version', action='version',
|
|
version=self._get_version(),
|
|
help='show zuul version')
|
|
return parser
|
|
|
|
def addSubCommands(self, parser, commands):
|
|
# Add a list of commandsocket.Command items to the parser
|
|
subparsers = parser.add_subparsers(
|
|
title='Online commands',
|
|
description=('The following commands may be used to affect '
|
|
'the running process.'),
|
|
)
|
|
for command in commands:
|
|
self.commands[command.name] = command
|
|
cmd = subparsers.add_parser(
|
|
command.name, help=command.help)
|
|
cmd.set_defaults(command=command.name)
|
|
for arg in command.args:
|
|
cmd.add_argument(arg.name,
|
|
help=arg.help,
|
|
default=arg.default)
|
|
|
|
def handleCommands(self):
|
|
command_name = getattr(self.args, 'command', None)
|
|
if command_name in self.commands:
|
|
command = self.commands[self.args.command]
|
|
command_str = command.name
|
|
command_args = [getattr(self.args, arg.name)
|
|
for arg in command.args]
|
|
if command_args:
|
|
command_str += ' ' + json.dumps(command_args)
|
|
self.sendCommand(command_str)
|
|
sys.exit(0)
|
|
|
|
def parseArguments(self, args=None):
|
|
parser = self.createParser()
|
|
self.args = parser.parse_args(args)
|
|
|
|
if getattr(self.args, 'foreground', False):
|
|
self.args.nodaemon = True
|
|
else:
|
|
self.args.nodaemon = False
|
|
|
|
if getattr(self.args, 'command', None):
|
|
self.args.nodaemon = True
|
|
|
|
return parser
|
|
|
|
def readConfig(self):
|
|
safe_env = {
|
|
k: v for k, v in os.environ.items()
|
|
if k.startswith('ZUUL_')
|
|
}
|
|
self.config = configparser.ConfigParser(safe_env)
|
|
if self.args.config:
|
|
locations = [self.args.config]
|
|
else:
|
|
locations = ['/etc/zuul/zuul.conf',
|
|
'~/zuul.conf']
|
|
for fp in locations:
|
|
if os.path.exists(os.path.expanduser(fp)):
|
|
self.config.read(os.path.expanduser(fp))
|
|
return
|
|
raise Exception("Unable to locate config file in %s" % locations)
|
|
|
|
def setup_logging(self, section, parameter):
|
|
if self.config.has_option(section, parameter):
|
|
fp = os.path.expanduser(self.config.get(section, parameter))
|
|
logging_config = logconfig.load_config(fp)
|
|
else:
|
|
# If someone runs in the foreground and doesn't give a logging
|
|
# config, leave the config set to emit to stdout.
|
|
if hasattr(self.args, 'nodaemon') and self.args.nodaemon:
|
|
logging_config = logconfig.ServerLoggingConfig()
|
|
else:
|
|
# Setting a server value updates the defaults to use
|
|
# WatchedFileHandler on /var/log/zuul/{server}-debug.log
|
|
# and /var/log/zuul/{server}.log
|
|
logging_config = logconfig.ServerLoggingConfig(server=section)
|
|
if hasattr(self.args, 'debug') and self.args.debug:
|
|
logging_config.setDebug()
|
|
logging_config.apply()
|
|
|
|
def configure_connections(self, source_only=False,
|
|
require_sql=False, check_bwrap=False):
|
|
self.connections = zuul.lib.connections.ConnectionRegistry(
|
|
check_bwrap=check_bwrap)
|
|
self.connections.configure(self.config, source_only, require_sql)
|
|
|
|
|
|
class ZuulDaemonApp(ZuulApp, metaclass=abc.ABCMeta):
|
|
def createParser(self):
|
|
parser = super(ZuulDaemonApp, self).createParser()
|
|
parser.add_argument('-d', dest='debug', action='store_true',
|
|
help='enable debug log')
|
|
parser.add_argument('-f', dest='foreground', action='store_true',
|
|
help='run in foreground')
|
|
return parser
|
|
|
|
def getPidFile(self):
|
|
pid_fn = get_default(self.config, self.app_name, 'pidfile',
|
|
'/var/run/zuul/%s.pid' % self.app_name,
|
|
expand_user=True)
|
|
return pid_fn
|
|
|
|
@abc.abstractmethod
|
|
def run(self):
|
|
"""
|
|
This is the main run method of the application.
|
|
"""
|
|
pass
|
|
|
|
def setup_logging(self, section, parameter):
|
|
super(ZuulDaemonApp, self).setup_logging(section, parameter)
|
|
from zuul.version import release_string
|
|
log = logging.getLogger(
|
|
"zuul.{section}".format(section=section.title()))
|
|
log.debug(
|
|
"Configured logging: {version}".format(
|
|
version=release_string))
|
|
|
|
def main(self):
|
|
self.parseArguments()
|
|
self.readConfig()
|
|
|
|
pid_fn = self.getPidFile()
|
|
pid = pid_file_module.TimeoutPIDLockFile(pid_fn, 10)
|
|
|
|
# Early register the stack dump handler for all zuul apps. This makes
|
|
# it possible to also gather stack dumps during startup hangs.
|
|
signal.signal(signal.SIGUSR2, stack_dump_handler)
|
|
|
|
if self.args.nodaemon:
|
|
self.run()
|
|
else:
|
|
# Exercise the pidfile before we do anything else (including
|
|
# logging or daemonizing)
|
|
with pid:
|
|
pass
|
|
|
|
with daemon.DaemonContext(pidfile=pid, umask=0o022):
|
|
self.run()
|
|
|
|
def sendCommand(self, cmd):
|
|
command_socket = get_default(
|
|
self.config, self.app_name, 'command_socket',
|
|
'/var/lib/zuul/%s.socket' % self.app_name)
|
|
s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
|
|
s.connect(command_socket)
|
|
cmd = '%s\n' % cmd
|
|
s.sendall(cmd.encode('utf8'))
|