zuul/zuul/ansible/logconfig.py

254 lines
7.4 KiB
Python

# Copyright 2017 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import abc
import copy
import logging.config
import json
import os
import yaml
_DEFAULT_JOB_LOGGING_CONFIG = {
'version': 1,
'formatters': {
'plain': {'format': '%(message)s'},
'result': {'format': 'RESULT %(message)s'},
},
'handlers': {
'console': {
'class': 'logging.StreamHandler',
'stream': 'ext://sys.stdout',
'level': 'INFO',
'formatter': 'plain',
},
'result': {
# result is returned to subprocess stdout and is used to pass
# control information from the callback back to the executor
'class': 'logging.StreamHandler',
'stream': 'ext://sys.stdout',
'level': 'INFO',
'formatter': 'result',
},
'jobfile': {
# used by executor to emit log file
'class': 'logging.FileHandler',
'level': 'INFO',
'formatter': 'plain',
},
},
'loggers': {
'zuul.executor.ansible.result': {
'handlers': ['result'],
'level': 'INFO',
},
'zuul.executor.ansible': {
'handlers': ['jobfile'],
'level': 'INFO',
},
'sqlalchemy.engine': {
'handlers': ['console'],
'level': 'WARN',
},
'alembic': {
'handlers': ['console'],
'level': 'WARN',
},
},
'root': {'handlers': []},
}
_DEFAULT_SERVER_LOGGING_CONFIG = {
'version': 1,
'formatters': {
'simple': {
'class': 'zuul.lib.logutil.MultiLineFormatter',
'format': '%(asctime)s %(levelname)s %(name)s: %(message)s'
},
},
'handlers': {
'console': {
# Used for printing to stdout
'class': 'logging.StreamHandler',
'stream': 'ext://sys.stdout',
'level': 'INFO',
'formatter': 'simple',
},
},
'loggers': {
'zuul': {
'handlers': ['console'],
'level': 'INFO',
},
'zuul.GerritConnection.io': {
'handlers': ['console'],
'level': 'WARN',
},
'sqlalchemy.engine': {
'handlers': ['console'],
'level': 'WARN',
},
'gerrit': {
'handlers': ['console'],
'level': 'INFO',
},
'alembic.runtime.migration': {
'handlers': ['console'],
'level': 'INFO',
},
'alembic': {
'handlers': ['console'],
'level': 'WARN',
},
'cachecontrol': {
'handlers': ['console'],
'level': 'INFO',
},
'cherrypy': {
'handlers': ['console'],
'level': 'WARN',
},
},
'root': {'handlers': []},
}
_DEFAULT_SERVER_FILE_HANDLERS = {
'normal': {
# Used for writing normal log files
'class': 'logging.handlers.WatchedFileHandler',
# This will get set to something real by DictLoggingConfig.server
'filename': '/var/log/zuul/{server}.log',
'level': 'INFO',
'formatter': 'simple',
},
}
def _read_config_file(filename: str):
if not os.path.exists(filename):
raise ValueError("Unable to read logging config file at %s" % filename)
if os.path.splitext(filename)[1] in ('.yml', '.yaml', '.json'):
with open(filename, 'r') as f:
return yaml.safe_load(f)
return filename
def load_config(filename: str):
config = _read_config_file(filename)
if isinstance(config, dict):
return DictLoggingConfig(config)
return FileLoggingConfig(filename)
def load_job_config(filename: str):
return JobLoggingConfig(_read_config_file(filename))
class LoggingConfig(object, metaclass=abc.ABCMeta):
@abc.abstractmethod
def apply(self):
"""Apply the config information to the current logging config."""
class DictLoggingConfig(LoggingConfig, metaclass=abc.ABCMeta):
def __init__(self, config):
self._config = config
def apply(self):
logging.config.dictConfig(self._config)
def writeJson(self, filename: str):
with open(filename, 'w') as f:
f.write(json.dumps(self._config, indent=2))
class JobLoggingConfig(DictLoggingConfig):
def __init__(self, config=None, job_output_file=None):
if not config:
config = copy.deepcopy(_DEFAULT_JOB_LOGGING_CONFIG)
super(JobLoggingConfig, self).__init__(config=config)
if job_output_file:
self.job_output_file = job_output_file
def setDebug(self):
# Set the level for zuul.executor.ansible to DEBUG
self._config['loggers']['zuul.executor.ansible']['level'] = 'DEBUG'
@property
def job_output_file(self) -> str:
return self._config['handlers']['jobfile']['filename']
@job_output_file.setter
def job_output_file(self, filename: str):
self._config['handlers']['jobfile']['filename'] = filename
class ServerLoggingConfig(DictLoggingConfig):
def __init__(self, config=None, server=None):
if not config:
config = copy.deepcopy(_DEFAULT_SERVER_LOGGING_CONFIG)
super(ServerLoggingConfig, self).__init__(config=config)
if server:
self.server = server
@property
def server(self):
return self._server
@server.setter
def server(self, server):
self._server = server
# Add the normal file handler. It's not included in the default
# config above because we're templating out the filename. Also, we
# only want to add the handler if we're actually going to use it.
for name, handler in _DEFAULT_SERVER_FILE_HANDLERS.items():
server_handler = copy.deepcopy(handler)
server_handler['filename'] = server_handler['filename'].format(
server=server)
self._config['handlers'][name] = server_handler
# Change everything configured to write to stdout to write to
# log files instead.
for logger in self._config['loggers'].values():
if logger['handlers'] == ['console']:
logger['handlers'] = ['normal']
def setDebug(self):
# Change level from INFO to DEBUG
for section in ('handlers', 'loggers'):
for handler in self._config[section].values():
if handler.get('level') == 'INFO':
handler['level'] = 'DEBUG'
class FileLoggingConfig(LoggingConfig):
def __init__(self, filename):
self._filename = filename
def apply(self):
logging.config.fileConfig(self._filename)
if __name__ == '__main__':
# Use this to emit a working logging output for testing zuul_stream
# locally.
config = JobLoggingConfig(
job_output_file=os.path.abspath('job-output.txt'))
config.writeJson('logging.json')