openstack/kolla-ansible
Code Issues Proposed changes

Performance: use a single config file for fluentd

Browse Source 
Currently we generate multiple fluentd configuration files for inputs,
filters, formatters and outputs.
These are then included from the main td-agent.conf configuration file.
With a large number of hosts, this can take a long time to template.

Benchmarking of templating is available at [1].

This change switches to a single fluentd configuration file, with the
include done locally. For the default template files included with Kolla
Ansible we use Jinja includes, but this does not work with templates in
a different directory. We therefore use the Ansible template lookup
plugin, which has a slightly higher overhead than a jinja include, but
far lower than generating multiple templates. This should drastically
improve the performance of this task.

[1] https://github.com/stackhpc/ansible-scaling/blob/master/doc/template.md

Partially-Implements: blueprint performance-improvements

Change-Id: Ia8623be0aa861fea3e54d2c9e1c971dfd8e3afa9
This commit is contained in:
Mark Goddard 2020-07-02 18:17:10 +01:00
parent f329af7dfa
commit 56a07702bc
7 changed files with 275 additions and 211 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
ansible/roles/common/filter_plugins/filters.py Normal file
View File

@ -0,0 +1,22 @@
# Copyright (c) 2020 StackHPC Ltd.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from kolla_ansible import fluentd_filters
class FilterModule(object):
"""Service filters."""
def filters(self):
return fluentd_filters.get_filters()

239
ansible/roles/common/tasks/config.yml
View File

@ -18,10 +18,6 @@
- service_name: "fluentd"
paths:
- "fluentd"
- "fluentd/input"
- "fluentd/output"
- "fluentd/format"
- "fluentd/filter"
- service_name: "kolla-toolbox"
paths:
- "kolla-toolbox"
@ -78,46 +74,22 @@
delegate_to: localhost
when: common_services.fluentd | service_enabled_and_mapped_to_host
- name: Copying over fluentd input config files
vars:
customised_input_files: "{{ find_custom_fluentd_inputs.files | map(attribute='path') | map('basename') | list }}"
template:
src: "conf/input/{{ item }}.conf.j2"
dest: "{{ node_config_directory }}/fluentd/input/{{ item }}.conf"
mode: "0660"
become: true
when:
- common_services.fluentd | service_enabled_and_mapped_to_host
- item ~ '.conf' not in customised_input_files
with_items:
- "00-global"
- "01-syslog"
- "02-mariadb"
- "03-rabbitmq"
- "04-openstack-wsgi"
- "05-libvirt"
- "06-zookeeper"
- "07-kafka"
- "09-monasca"
notify:
- Restart fluentd container
- name: Find custom fluentd filter config files
find:
path: "{{ node_custom_config }}/fluentd/filter"
pattern: "*.conf"
run_once: True
register: find_custom_fluentd_filters
delegate_to: localhost
when: common_services.fluentd | service_enabled_and_mapped_to_host
- name: Copying over custom fluentd input config files
template:
src: "{{ item.path }}"
dest: "{{ node_config_directory }}/fluentd/input/{{ item.path | basename }}"
mode: "0660"
when:
- common_services.fluentd | service_enabled_and_mapped_to_host
with_items: "{{ find_custom_fluentd_inputs.files }}"
notify:
- Restart fluentd container
- name: Determine whether logs should be forwarded directly to Elasticsearch
set_fact:
log_direct_to_elasticsearch: "{{ ( enable_elasticsearch | bool or
( elasticsearch_address != kolla_internal_vip_address )) and
not enable_monasca | bool }}"
- name: Find custom fluentd format config files
find:
path: "{{ node_custom_config }}/fluentd/format"
pattern: "*.conf"
run_once: True
register: find_custom_fluentd_formats
delegate_to: localhost
when:
- common_services.fluentd | service_enabled_and_mapped_to_host
@ -131,146 +103,57 @@
when:
- common_services.fluentd | service_enabled_and_mapped_to_host
- name: Copying over fluentd output config files
vars:
customised_output_files: "{{ find_custom_fluentd_outputs.files | map(attribute='path') | map('basename') | list }}"
template:
src: "conf/output/{{ item.name }}.conf.j2"
dest: "{{ node_config_directory }}/fluentd/output/{{ item.name }}.conf"
mode: "0660"
become: true
when:
- common_services.fluentd | service_enabled_and_mapped_to_host
- item.enabled | bool
- item.name ~ '.conf' not in customised_output_files
with_items:
- name: "00-local"
enabled: true
- name: "01-es"
enabled: "{{ log_direct_to_elasticsearch }}"
- name: "02-monasca"
enabled: "{{ enable_monasca | bool }}"
notify:
- Restart fluentd container
- name: Removing stale output config files
file:
path: "{{ node_config_directory }}/fluentd/output/{{ item.name }}.conf"
state: "absent"
become: true
when:
- common_services.fluentd | service_enabled_and_mapped_to_host
- item.disable | bool
with_items:
- name: "02-monasca"
disable: "{{ not enable_monasca | bool }}"
- name: "01-es"
disable: "{{ not log_direct_to_elasticsearch }}"
notify:
- Restart fluentd container
- name: Copying over custom fluentd output config files
template:
src: "{{ item.path }}"
dest: "{{ node_config_directory }}/fluentd/output/{{ item.path | basename }}"
mode: "0660"
become: true
when:
- common_services.fluentd | service_enabled_and_mapped_to_host
with_items: "{{ find_custom_fluentd_outputs.files }}"
notify:
- Restart fluentd container
- name: Find custom fluentd format config files
find:
path: "{{ node_custom_config }}/fluentd/format"
pattern: "*.conf"
run_once: True
register: find_custom_fluentd_format
delegate_to: localhost
when:
- common_services.fluentd | service_enabled_and_mapped_to_host
- name: Copying over fluentd format config files
vars:
customised_format_files: "{{ find_custom_fluentd_format.files | map(attribute='path') | map('basename') | list }}"
template:
src: "conf/format/{{ item }}.conf.j2"
dest: "{{ node_config_directory }}/fluentd/format/{{ item }}.conf"
mode: "0660"
become: true
with_items:
- "apache_access"
- "wsgi_access"
when:
- common_services.fluentd | service_enabled_and_mapped_to_host
- item ~ '.conf' not in customised_format_files
notify:
- Restart fluentd container
- name: Copying over custom fluentd format config files
template:
src: "{{ item.path }}"
dest: "{{ node_config_directory }}/fluentd/format/{{ item.path | basename }}"
mode: "0660"
when:
- common_services.fluentd | service_enabled_and_mapped_to_host
with_items: "{{ find_custom_fluentd_format.files }}"
notify:
- Restart fluentd container
- name: Find custom fluentd filter config files
find:
path: "{{ node_custom_config }}/fluentd/filter"
pattern: "*.conf"
run_once: True
register: find_custom_fluentd_filters
delegate_to: localhost
when: common_services.fluentd | service_enabled_and_mapped_to_host
- name: Copying over fluentd filter config files
vars:
customised_filter_files: "{{ find_custom_fluentd_filters.files | map(attribute='path') | map('basename') | list }}"
fluentd_version: "{{ fluentd_labels.images.0.ContainerConfig.Labels.fluentd_version | default('0.12') }}"
template:
src: "conf/filter/{{ item.src }}.conf.j2"
dest: "{{ node_config_directory }}/fluentd/filter/{{ item.dest }}.conf"
mode: "0660"
become: true
with_items:
- src: 00-record_transformer
dest: 00-record_transformer
- src: "{{ '01-rewrite-0.14' if fluentd_version == '0.14' else '01-rewrite-0.12' }}"
dest: 01-rewrite
- src: 02-parser
dest: 02-parser
when:
- common_services.fluentd | service_enabled_and_mapped_to_host
- item.src ~ '.conf' not in customised_filter_files
notify:
- Restart fluentd container
- name: Copying over custom fluentd filter config files
template:
src: "{{ item.path }}"
dest: "{{ node_config_directory }}/fluentd/filter/{{ item.path | basename }}"
mode: "0660"
become: true
with_items: "{{ find_custom_fluentd_filters.files }}"
when:
- common_services.fluentd | service_enabled_and_mapped_to_host
notify:
- Restart fluentd container
- name: Copying over td-agent.conf
vars:
log_direct_to_elasticsearch: >-
{{ ( enable_elasticsearch | bool or
( elasticsearch_address != kolla_internal_vip_address )) and
not enable_monasca | bool }}
fluentd_version: "{{ fluentd_labels.images.0.ContainerConfig.Labels.fluentd_version | default('0.12') }}"
# Inputs
fluentd_input_files: "{{ default_input_files | customise_fluentd(customised_input_files) }}"
default_input_files:
- "conf/input/00-global.conf.j2"
- "conf/input/01-syslog.conf.j2"
- "conf/input/02-mariadb.conf.j2"
- "conf/input/03-rabbitmq.conf.j2"
- "conf/input/04-openstack-wsgi.conf.j2"
- "conf/input/05-libvirt.conf.j2"
- "conf/input/06-zookeeper.conf.j2"
- "conf/input/07-kafka.conf.j2"
- "conf/input/09-monasca.conf.j2"
customised_input_files: "{{ find_custom_fluentd_inputs.files | map(attribute='path') | list }}"
# Filters
fluentd_filter_files: "{{ default_filter_files | customise_fluentd(customised_filter_files) }}"
default_filter_files:
- "conf/filter/00-record_transformer.conf.j2"
- "conf/filter/{{ '01-rewrite-0.14' if fluentd_version == '0.14' else '01-rewrite-0.12' }}.conf.j2"
- "conf/filter/02-parser.conf.j2"
customised_filter_files: "{{ find_custom_fluentd_filters.files | map(attribute='path') | list }}"
# Formats
fluentd_format_files: "{{ default_format_files | customise_fluentd(customised_format_files) }}"
default_format_files:
- "conf/format/apache_access.conf.j2"
- "conf/format/wsgi_access.conf.j2"
customised_format_files: "{{ find_custom_fluentd_formats.files | map(attribute='path') | list }}"
# Outputs
fluentd_output_files: "{{ default_output_files_enabled | customise_fluentd(customised_output_files) }}"
default_output_files_enabled: "{{ default_output_files | selectattr('enabled') | map(attribute='name') | list }}"
default_output_files:
- name: "conf/output/00-local.conf.j2"
enabled: true
- name: "conf/output/01-es.conf.j2"
enabled: "{{ log_direct_to_elasticsearch }}"
- name: "conf/output/02-monasca.conf.j2"
enabled: "{{ enable_monasca | bool }}"
customised_output_files: "{{ find_custom_fluentd_outputs.files | map(attribute='path') | list }}"
template:
src: "td-agent.conf.j2"
dest: "{{ node_config_directory }}/{{ item }}/td-agent.conf"
dest: "{{ node_config_directory }}/fluentd/td-agent.conf"
mode: "0660"
become: true
with_items:
- "fluentd"
when: common_services.fluentd | service_enabled_and_mapped_to_host
when:
- common_services.fluentd | service_enabled_and_mapped_to_host
notify:
- Restart fluentd container

31
ansible/roles/common/templates/fluentd.json.j2
View File

@ -1,15 +1,6 @@
{% set fluentd_user = fluentd_binary %}
{% set fluentd_dir = '/etc/' ~ fluentd_binary %}
{%- macro config_directory(dir) -%}
{
"source": "{{ container_config_directory }}/{{ dir }}",
"dest": "{{ fluentd_dir }}/{{ dir }}",
"owner": "{{ fluentd_user }}",
"perm": "0600"
}
{%- endmacro -%}
{% if fluentd_binary == 'fluentd' %}
{% set fluentd_conf = 'fluent.conf' %}
{% if kolla_base_distro in ['ubuntu', 'debian'] %}
@ -22,14 +13,6 @@
{% set fluentd_cmd = '/usr/sbin/td-agent' %}
{% endif %}
{%- macro config_directory_permissions(dir) -%}
{
"path": "{{ fluentd_dir }}/{{ dir }}",
"owner": "{{ fluentd_user }}:{{ fluentd_user }}",
"perm": "0700"
}
{%- endmacro -%}
{
"command": "{{ fluentd_cmd }} -o /var/log/kolla/fluentd/fluentd.log",
"config_files": [
@ -38,12 +21,7 @@
"dest": "{{ fluentd_dir }}/{{ fluentd_conf }}",
"owner": "{{ fluentd_user }}",
"perm": "0600"
},
{# Copy all files in the following directories #}
{{ config_directory("input") }},
{{ config_directory("filter") }},
{{ config_directory("format") }},
{{ config_directory("output") }}
}
],
"permissions": [
{
@ -65,12 +43,7 @@
"path": "/var/lib/fluentd/data",
"owner": "{{ fluentd_user }}:{{ fluentd_user }}",
"recurse": true
},
{# Allow Fluentd to read configuration from folders #}
{{ config_directory_permissions("input") }},
{{ config_directory_permissions("filter") }},
{{ config_directory_permissions("format") }},
{{ config_directory_permissions("output") }}
}
]
}

49
ansible/roles/common/templates/td-agent.conf.j2
View File

@ -1,4 +1,45 @@
@include input/*.conf
@include filter/*.conf
@include format/*.conf
@include output/*.conf
#jinja2: trim_blocks: False
{# Ansible restricts Jinja includes to the same directory or subdirectory of a
template. To support customised configuration outside of this path we use
the template lookup plugin. Jinja includes have a lower overhead, so we use
those where possible. #}
# Inputs
{%- for path in fluentd_input_files %}
# Included from {{ path }}:
{%- if path.startswith('/') %}
{{ lookup('template', path) }}
{%- else %}
{% include path %}
{%- endif %}
{%- endfor %}
# Filters
{%- for path in fluentd_filter_files %}
# Included from {{ path }}:
{%- if path.startswith('/') %}
{{ lookup('template', path) }}
{%- else %}
{% include path %}
{%- endif %}
{%- endfor %}
# Formats
{%- for path in fluentd_format_files %}
# Included from {{ path }}:
{%- if path.startswith('/') %}
{{ lookup('template', path) }}
{%- else %}
{% include path %}
{%- endif %}
{%- endfor %}
# Outputs
{%- for path in fluentd_output_files %}
# Included from {{ path }}:
{%- if path.startswith('/') %}
{{ lookup('template', path) }}
{%- else %}
{% include path %}
{%- endif %}
{%- endfor %}

44
kolla_ansible/fluentd_filters.py Normal file
View File

@ -0,0 +1,44 @@
# Copyright (c) 2020 StackHPC Ltd.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import os.path
def customise_fluentd(default_paths, customised_paths):
"""Return a sorted list of templates for fluentd.
:param default_paths: Iterable of default template paths.
:param customised_paths: Iterable of customised template paths.
:returns: A sorted combined list of template paths.
"""
def _basename_no_ext(path):
"""Return the basename of a path, stripping off any extension."""
return os.path.splitext(os.path.basename(path))[0]
customised_file_names = {os.path.basename(f) for f in customised_paths}
# Starting with the default paths, remove any that have been overridden,
# ignoring the .j2 extension of default paths.
result = {f for f in default_paths
if _basename_no_ext(f) not in customised_file_names}
# Add all customised paths.
result.update(customised_paths)
# Sort by the basename of the paths.
return sorted(result, key=os.path.basename)
def get_filters():
return {
"customise_fluentd": customise_fluentd,
}

96
kolla_ansible/tests/unit/test_fluentd_filters.py Normal file
View File

@ -0,0 +1,96 @@
# Copyright (c) 2020 StackHPC Ltd.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import unittest
from kolla_ansible.fluentd_filters import customise_fluentd
class TestFilters(unittest.TestCase):
def test_customise_fluentd_no_files(self):
default_files = [
]
customised_files = [
]
expected = [
]
result = customise_fluentd(default_files, customised_files)
self.assertEqual(expected, result)
def test_customise_fluentd_no_customised_files(self):
default_files = [
"foo/bar.conf.j2"
]
customised_files = [
]
expected = [
"foo/bar.conf.j2"
]
result = customise_fluentd(default_files, customised_files)
self.assertEqual(expected, result)
def test_customise_fluentd_no_default_files(self):
default_files = [
]
customised_files = [
"foo/bar.conf"
]
expected = [
"foo/bar.conf"
]
result = customise_fluentd(default_files, customised_files)
self.assertEqual(expected, result)
def test_customise_fluentd_both(self):
default_files = [
"foo/bar.conf.j2"
]
customised_files = [
"baz/qux.conf"
]
expected = [
"foo/bar.conf.j2",
"baz/qux.conf"
]
result = customise_fluentd(default_files, customised_files)
self.assertEqual(expected, result)
def test_customise_fluentd_override(self):
default_files = [
"foo/bar.conf.j2"
]
customised_files = [
"baz/bar.conf"
]
expected = [
"baz/bar.conf"
]
result = customise_fluentd(default_files, customised_files)
self.assertEqual(expected, result)
def test_customise_fluentd_both_with_override(self):
default_files = [
"foo/bar.conf.j2",
"baz/qux.conf.j2"
]
customised_files = [
"baz/bar.conf"
]
expected = [
"baz/bar.conf",
"baz/qux.conf.j2"
]
result = customise_fluentd(default_files, customised_files)
self.assertEqual(expected, result)

5
releasenotes/notes/fluentd-single-config-d5ae95fecbfb6e3e.yaml Normal file
View File

@ -0,0 +1,5 @@
---
features:
- |
Improves performance of the ``common`` role by generating all fluentd
configuration in a single file.