Remove per-image logging from nodepool-builder logging config

Change Ia7415d2fbbb320f8eddc4e46c3a055414df5f997 removed per-image
logging from nodepool; it now keeps a series of named log files.  The
"image" logger is removed, along with all the specific build logs.

I972a66f9c153f52df934d25ab2f77596bb2e5429 removed the
image_log_document_root argument; which wasn't doing anything in
nodepool::builder, but was actually being used to fill-out the
builder-logging.conf file.  Thus we need this to not have a bad path
in the config file.

Change-Id: I6fdc3b622454b069f3ad2cf42da584d8df23a110
This commit is contained in:
Ian Wienand 2018-02-26 14:12:35 +11:00
parent d51edbf876
commit 53ac2cfeae
2 changed files with 2 additions and 597 deletions

View File

@ -1,27 +1,8 @@
#
# THIS FILE HAS BEEN AUTOGENERATED
# Regenerate it with tools/nodepool_log_config.py
#
[loggers]
keys=root,nodepool,requests,shade,image,kazoo,
dib_centos-7, dib_debian-jessie, dib_fedora-26, dib_fedora-27,
dib_gentoo, dib_opensuse-423, dib_ubuntu-trusty, dib_ubuntu-xenial,
citycloud-kna1_upload, citycloud-la1_upload, citycloud-lon1_upload,
citycloud-sto2_upload, ovh-bhs1_upload, ovh-gra1_upload,
inap-mtl01_upload, rax-dfw_upload, rax-iad_upload, rax-ord_upload,
vexxhost-ca-ymq-1_upload
keys=root,nodepool,requests,shade,kazoo
[handlers]
keys=console,debug,normal,image,
dib_centos-7, dib_debian-jessie, dib_fedora-26, dib_fedora-27,
dib_gentoo, dib_opensuse-423, dib_ubuntu-trusty, dib_ubuntu-xenial,
citycloud-kna1_upload, citycloud-la1_upload, citycloud-lon1_upload,
citycloud-sto2_upload, ovh-bhs1_upload, ovh-gra1_upload,
inap-mtl01_upload, rax-dfw_upload, rax-iad_upload, rax-ord_upload,
vexxhost-ca-ymq-1_upload
keys=console,debug,normal
[formatters]
keys=simple
@ -45,12 +26,6 @@ level=DEBUG
handlers=debug,normal
qualname=nodepool
[logger_image]
level=DEBUG
handlers=image
qualname=nodepool.image.build
propagate=0
[logger_kazoo]
level=INFO
handlers=debug,normal
@ -75,244 +50,6 @@ class=logging.handlers.TimedRotatingFileHandler
formatter=simple
args=('/var/log/nodepool/nodepool-builder.log', 'H', 8, 30,)
[handler_image]
level=DEBUG
class=logging.handlers.TimedRotatingFileHandler
formatter=simple
args=('<%= @image_log_document_root %>/image.log', 'H', 8, 30,)
[formatter_simple]
format=%(asctime)s %(levelname)s %(name)s: %(message)s
datefmt=
# ==== loggers and handlers ====
[logger_dib_centos-7]
level=DEBUG
handlers=dib_centos-7
qualname=nodepool.image.build.centos-7
propagate=0
[handler_dib_centos-7]
level=DEBUG
class=logging.handlers.TimedRotatingFileHandler
formatter=simple
args=('<%= @image_log_document_root %>/dib.centos-7.log', 'H', 8, 30,)
[logger_dib_debian-jessie]
level=DEBUG
handlers=dib_debian-jessie
qualname=nodepool.image.build.debian-jessie
propagate=0
[handler_dib_debian-jessie]
level=DEBUG
class=logging.handlers.TimedRotatingFileHandler
formatter=simple
args=('<%= @image_log_document_root %>/dib.debian-jessie.log', 'H', 8, 30,)
[logger_dib_fedora-26]
level=DEBUG
handlers=dib_fedora-26
qualname=nodepool.image.build.fedora-26
propagate=0
[handler_dib_fedora-26]
level=DEBUG
class=logging.handlers.TimedRotatingFileHandler
formatter=simple
args=('<%= @image_log_document_root %>/dib.fedora-26.log', 'H', 8, 30,)
[logger_dib_fedora-27]
level=DEBUG
handlers=dib_fedora-27
qualname=nodepool.image.build.fedora-27
propagate=0
[handler_dib_fedora-27]
level=DEBUG
class=logging.handlers.TimedRotatingFileHandler
formatter=simple
args=('<%= @image_log_document_root %>/dib.fedora-27.log', 'H', 8, 30,)
[logger_dib_gentoo]
level=DEBUG
handlers=dib_gentoo
qualname=nodepool.image.build.gentoo
propagate=0
[handler_dib_gentoo]
level=DEBUG
class=logging.handlers.TimedRotatingFileHandler
formatter=simple
args=('<%= @image_log_document_root %>/dib.gentoo.log', 'H', 8, 30,)
[logger_dib_opensuse-423]
level=DEBUG
handlers=dib_opensuse-423
qualname=nodepool.image.build.opensuse-423
propagate=0
[handler_dib_opensuse-423]
level=DEBUG
class=logging.handlers.TimedRotatingFileHandler
formatter=simple
args=('<%= @image_log_document_root %>/dib.opensuse-423.log', 'H', 8, 30,)
[logger_dib_ubuntu-trusty]
level=DEBUG
handlers=dib_ubuntu-trusty
qualname=nodepool.image.build.ubuntu-trusty
propagate=0
[handler_dib_ubuntu-trusty]
level=DEBUG
class=logging.handlers.TimedRotatingFileHandler
formatter=simple
args=('<%= @image_log_document_root %>/dib.ubuntu-trusty.log', 'H', 8, 30,)
[logger_dib_ubuntu-xenial]
level=DEBUG
handlers=dib_ubuntu-xenial
qualname=nodepool.image.build.ubuntu-xenial
propagate=0
[handler_dib_ubuntu-xenial]
level=DEBUG
class=logging.handlers.TimedRotatingFileHandler
formatter=simple
args=('<%= @image_log_document_root %>/dib.ubuntu-xenial.log', 'H', 8, 30,)
[logger_citycloud-kna1_upload]
level=DEBUG
handlers=citycloud-kna1_upload
qualname=nodepool.image.upload.citycloud-kna1
propagate=0
[handler_citycloud-kna1_upload]
level=DEBUG
class=logging.handlers.TimedRotatingFileHandler
formatter=simple
args=('<%= @upload_log_document_root %>/citycloud-kna1.log', 'H', 8, 30,)
[logger_citycloud-la1_upload]
level=DEBUG
handlers=citycloud-la1_upload
qualname=nodepool.image.upload.citycloud-la1
propagate=0
[handler_citycloud-la1_upload]
level=DEBUG
class=logging.handlers.TimedRotatingFileHandler
formatter=simple
args=('<%= @upload_log_document_root %>/citycloud-la1.log', 'H', 8, 30,)
[logger_citycloud-lon1_upload]
level=DEBUG
handlers=citycloud-lon1_upload
qualname=nodepool.image.upload.citycloud-lon1
propagate=0
[handler_citycloud-lon1_upload]
level=DEBUG
class=logging.handlers.TimedRotatingFileHandler
formatter=simple
args=('<%= @upload_log_document_root %>/citycloud-lon1.log', 'H', 8, 30,)
[logger_citycloud-sto2_upload]
level=DEBUG
handlers=citycloud-sto2_upload
qualname=nodepool.image.upload.citycloud-sto2
propagate=0
[handler_citycloud-sto2_upload]
level=DEBUG
class=logging.handlers.TimedRotatingFileHandler
formatter=simple
args=('<%= @upload_log_document_root %>/citycloud-sto2.log', 'H', 8, 30,)
[logger_ovh-bhs1_upload]
level=DEBUG
handlers=ovh-bhs1_upload
qualname=nodepool.image.upload.ovh-bhs1
propagate=0
[handler_ovh-bhs1_upload]
level=DEBUG
class=logging.handlers.TimedRotatingFileHandler
formatter=simple
args=('<%= @upload_log_document_root %>/ovh-bhs1.log', 'H', 8, 30,)
[logger_ovh-gra1_upload]
level=DEBUG
handlers=ovh-gra1_upload
qualname=nodepool.image.upload.ovh-gra1
propagate=0
[handler_ovh-gra1_upload]
level=DEBUG
class=logging.handlers.TimedRotatingFileHandler
formatter=simple
args=('<%= @upload_log_document_root %>/ovh-gra1.log', 'H', 8, 30,)
[logger_inap-mtl01_upload]
level=DEBUG
handlers=inap-mtl01_upload
qualname=nodepool.image.upload.inap-mtl01
propagate=0
[handler_inap-mtl01_upload]
level=DEBUG
class=logging.handlers.TimedRotatingFileHandler
formatter=simple
args=('<%= @upload_log_document_root %>/inap-mtl01.log', 'H', 8, 30,)
[logger_rax-dfw_upload]
level=DEBUG
handlers=rax-dfw_upload
qualname=nodepool.image.upload.rax-dfw
propagate=0
[handler_rax-dfw_upload]
level=DEBUG
class=logging.handlers.TimedRotatingFileHandler
formatter=simple
args=('<%= @upload_log_document_root %>/rax-dfw.log', 'H', 8, 30,)
[logger_rax-iad_upload]
level=DEBUG
handlers=rax-iad_upload
qualname=nodepool.image.upload.rax-iad
propagate=0
[handler_rax-iad_upload]
level=DEBUG
class=logging.handlers.TimedRotatingFileHandler
formatter=simple
args=('<%= @upload_log_document_root %>/rax-iad.log', 'H', 8, 30,)
[logger_rax-ord_upload]
level=DEBUG
handlers=rax-ord_upload
qualname=nodepool.image.upload.rax-ord
propagate=0
[handler_rax-ord_upload]
level=DEBUG
class=logging.handlers.TimedRotatingFileHandler
formatter=simple
args=('<%= @upload_log_document_root %>/rax-ord.log', 'H', 8, 30,)
[logger_vexxhost-ca-ymq-1_upload]
level=DEBUG
handlers=vexxhost-ca-ymq-1_upload
qualname=nodepool.image.upload.vexxhost-ca-ymq-1
propagate=0
[handler_vexxhost-ca-ymq-1_upload]
level=DEBUG
class=logging.handlers.TimedRotatingFileHandler
formatter=simple
args=('<%= @upload_log_document_root %>/vexxhost-ca-ymq-1.log', 'H', 8, 30,)

View File

@ -1,332 +0,0 @@
#!/usr/bin/env python
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""Generate a sample logging configuration file
use log_config_generator.generate_log_config() to generate a sample
logging configuration file.
The sample splits up the log output for general logging and image
builds and applys some sensible rotation defaults.
"""
import argparse
import logging
import textwrap
import yaml
# default paths and outputs
MODULES_PATH = '../modules/openstack_project/templates/nodepool'
# default the project-config repo path to the same folder as this repo
CONFIG_FILE = '../../project-config/nodepool/nodepool.yaml'
LOGGING_CONFIG_FILE = MODULES_PATH + '/nodepool-builder.logging.conf.erb'
LOG_DIR = '/var/log/nodepool'
IMAGE_LOG_DIR = '<%= @image_log_document_root %>'
UPLOAD_LOG_DIR = '<%= @upload_log_document_root %>'
_BASIC_FILE = """
#
# THIS FILE HAS BEEN AUTOGENERATED
# Regenerate it with tools/nodepool_log_config.py
#
[loggers]
keys=root,nodepool,requests,shade,image,kazoo,
%(logger_titles)s
[handlers]
keys=console,debug,normal,image,
%(handler_titles)s
[formatters]
keys=simple
[logger_root]
level=WARNING
handlers=console
[logger_requests]
level=WARNING
handlers=debug,normal
qualname=requests
[logger_shade]
level=WARNING
handlers=debug,normal
qualname=shade
[logger_nodepool]
level=DEBUG
handlers=debug,normal
qualname=nodepool
[logger_image]
level=DEBUG
handlers=image
qualname=nodepool.image.build
propagate=0
[logger_kazoo]
level=INFO
handlers=debug,normal
qualname=kazoo
propagate=0
[handler_console]
level=WARNING
class=StreamHandler
formatter=simple
args=(sys.stdout,)
[handler_debug]
level=DEBUG
class=logging.handlers.TimedRotatingFileHandler
formatter=simple
args=('%(log_dir)s/builder-debug.log', 'H', 8, 30,)
[handler_normal]
level=INFO
class=logging.handlers.TimedRotatingFileHandler
formatter=simple
args=('%(log_dir)s/nodepool-builder.log', 'H', 8, 30,)
[handler_image]
level=DEBUG
class=logging.handlers.TimedRotatingFileHandler
formatter=simple
args=('%(image_log_dir)s/image.log', 'H', 8, 30,)
[formatter_simple]
format=%%(asctime)s %%(levelname)s %%(name)s: %%(message)s
datefmt=
# ==== loggers and handlers ====
%(loggers_and_handlers)s
"""
_IMAGE_HANDLER = """
[handler_%(title)s]
level=DEBUG
class=logging.handlers.TimedRotatingFileHandler
formatter=simple
args=('%(image_log_dir)s/%(filename)s', 'H', 8, 30,)
"""
_IMAGE_LOGGER = """
[logger_%(title)s]
level=DEBUG
handlers=%(handler)s
qualname=nodepool.image.build.%(qualname)s
propagate=0
"""
_UPLOAD_HANDLER = """
[handler_%(title)s]
level=DEBUG
class=logging.handlers.TimedRotatingFileHandler
formatter=simple
args=('%(upload_log_dir)s/%(filename)s', 'H', 8, 30,)
"""
_UPLOAD_LOGGER = """
[logger_%(title)s]
level=DEBUG
handlers=%(handler)s
qualname=nodepool.image.upload.%(qualname)s
propagate=0
"""
def _get_providers_and_images(config):
ret = []
# diskimages have a special provider
if 'diskimages' in config:
for diskimage in config['diskimages']:
ret.append(('dib', diskimage['name']))
logging.debug("Added dib image: %s" % diskimage['name'])
return ret
def _get_upload_providers(config):
ret = []
for provider in config['providers']:
ret.append(provider['name'])
logging.debug("Added upload provider: %s" % provider['name'])
return ret
def _generate_image_logger_and_handler(image_log_dir, provider, image):
handler = _IMAGE_HANDLER % {
'image_log_dir': image_log_dir,
'title': '%s_%s' % (provider, image),
'filename': '%s.%s.log' % (provider, image),
}
logger = _IMAGE_LOGGER % {
'title': '%s_%s' % (provider, image),
'handler': '%s_%s' % (provider, image),
'qualname': '%s%s' % (provider + "." if provider != 'dib' else '',
image),
}
return {
'handler_title': '%s_%s' % (provider, image),
'logger_title': '%s_%s' % (provider, image),
'handler': handler,
'logger': logger,
}
def _generate_upload_logger_and_handler(upload_log_dir, provider):
handler = _UPLOAD_HANDLER % {
'upload_log_dir': upload_log_dir,
'title': '%s_upload' % (provider),
'filename': '%s.log' % (provider),
}
logger = _UPLOAD_LOGGER % {
'title': '%s_upload' % (provider),
'handler': '%s_upload' % (provider),
'qualname': '%s' % (provider),
}
return {
'handler_title': '%s_upload' % (provider),
'logger_title': '%s_upload' % (provider),
'handler': handler,
'logger': logger,
}
def generate_log_config(config, log_dir, image_log_dir,
upload_log_dir, output):
"""Generate a sample logging file
The logging output will have the correct formatters and handlers
to split all image-build logs out into separate files grouped by
provider. e.g.
providers:
- name: foo
...
- images:
- name: image1
...
- name: image2
...
- name: moo
...
- images:
- name: image1
...
- name: image2
...
Will result in log files (in `image_log_dir`) of foo.image1.log,
foo.image2.log, moo.image1.log, moo.image2.log
diskimage-builder built images will have special provider "dib"
:param config: input config file
:param log_dir: directory for main log file
:param image_log_dir: directory for image build logs
:param upload_log_dir: directory for upload logs
:param output: open file handle to output sample configuration to
"""
loggers_and_handlers = []
logging.debug("Reading config file %s" % config.name)
config = yaml.load(config)
# image loggers
for (provider, image) in _get_providers_and_images(config):
loggers_and_handlers.append(
_generate_image_logger_and_handler(image_log_dir, provider, image))
# provider loggers (for upload)
for provider in _get_upload_providers(config):
loggers_and_handlers.append(
_generate_upload_logger_and_handler(upload_log_dir, provider))
# output all loggers and handlers
logger_titles = []
handler_titles = []
image_loggers_and_handlers = ""
for item in loggers_and_handlers:
logger_titles.append(item['logger_title'])
handler_titles.append(item['handler_title'])
image_loggers_and_handlers += item['logger'] + item['handler']
# Final output
final_output = _BASIC_FILE % {
'log_dir': log_dir,
'image_log_dir': image_log_dir,
'logger_titles': "\n".join(textwrap.wrap(', '.join(logger_titles),
break_long_words=False,
break_on_hyphens=False,
subsequent_indent=' ')),
'handler_titles': "\n".join(textwrap.wrap(', '.join(handler_titles),
break_long_words=False,
break_on_hyphens=False,
subsequent_indent=' ')),
'loggers_and_handlers': image_loggers_and_handlers,
}
logging.debug("Writing output to %s" % output.name)
output.write(final_output)
output.flush()
logging.debug("Done!")
def main():
parser = argparse.ArgumentParser()
parser.add_argument('-d', '--debug', action='store_true',
help="Enable debugging")
parser.add_argument('-c', '--config', default=CONFIG_FILE,
help="Config file to read in "
"(default: %s)" % CONFIG_FILE,
type=argparse.FileType('r'))
parser.add_argument('-o', '--output', default=LOGGING_CONFIG_FILE,
help="Output file "
"(default: %s)" % LOGGING_CONFIG_FILE,
type=argparse.FileType('w'))
parser.add_argument('-l', '--log-dir', default=LOG_DIR,
help="Output directory for logs "
"(default: %s)" % LOG_DIR)
parser.add_argument('-i', '--image-log-dir', default=IMAGE_LOG_DIR,
help="Output directory for image logs "
"(default: %s)" % IMAGE_LOG_DIR.replace('%', '%%'))
parser.add_argument('-u', '--upload-log-dir', default=UPLOAD_LOG_DIR,
help="Output directory for upload logs "
"(default: %s)" % UPLOAD_LOG_DIR.replace('%', '%%'))
args = parser.parse_args()
logging.basicConfig(level=logging.DEBUG if args.debug else logging.INFO)
generate_log_config(args.config,
args.log_dir,
args.image_log_dir,
args.upload_log_dir,
args.output)
if __name__ == "__main__":
main()